{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can an RNN learn to keep a clock as it's state? No input, can an RNN follow a circle?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "1600000\n",
      "1700000\n",
      "1800000\n",
      "1900000\n",
      "2000000\n",
      "2100000\n",
      "2200000\n",
      "2300000\n",
      "2400000\n",
      "2500000\n",
      "2600000\n",
      "2700000\n",
      "2800000\n",
      "2900000\n",
      "3000000\n",
      "3100000\n",
      "3200000\n",
      "3300000\n",
      "3400000\n",
      "3500000\n",
      "3600000\n",
      "3700000\n",
      "3800000\n",
      "3900000\n",
      "4000000\n",
      "4100000\n",
      "4200000\n",
      "4300000\n",
      "4400000\n",
      "4500000\n",
      "4600000\n",
      "4700000\n",
      "4800000\n",
      "4900000\n",
      "5000000\n",
      "5100000\n",
      "5200000\n",
      "5300000\n",
      "5400000\n",
      "5500000\n",
      "5600000\n",
      "5700000\n",
      "5800000\n",
      "5900000\n",
      "6000000\n",
      "6100000\n",
      "6200000\n",
      "6300000\n",
      "6400000\n",
      "6500000\n",
      "6600000\n",
      "6700000\n",
      "6800000\n",
      "6900000\n",
      "7000000\n",
      "7100000\n",
      "7200000\n",
      "7300000\n",
      "7400000\n",
      "7500000\n",
      "7600000\n",
      "7700000\n",
      "7800000\n",
      "7900000\n",
      "8000000\n",
      "8100000\n",
      "8200000\n",
      "8300000\n",
      "8400000\n",
      "8500000\n",
      "8600000\n",
      "8700000\n",
      "8800000\n",
      "8900000\n",
      "9000000\n",
      "9100000\n",
      "9200000\n",
      "9300000\n",
      "9400000\n",
      "9500000\n",
      "9600000\n",
      "9700000\n",
      "9800000\n",
      "9900000\n",
      "10000000\n",
      "10100000\n",
      "10200000\n",
      "10300000\n",
      "10400000\n",
      "10500000\n",
      "10600000\n",
      "10700000\n",
      "10800000\n",
      "10900000\n",
      "11000000\n",
      "11100000\n",
      "11200000\n",
      "11300000\n",
      "11400000\n",
      "11500000\n",
      "11600000\n",
      "11700000\n",
      "11800000\n",
      "11900000\n",
      "12000000\n",
      "12100000\n",
      "12200000\n",
      "12300000\n",
      "12400000\n",
      "12500000\n",
      "12600000\n",
      "12700000\n",
      "12800000\n",
      "12900000\n",
      "13000000\n",
      "13100000\n",
      "13200000\n",
      "13300000\n",
      "13400000\n",
      "13500000\n",
      "13600000\n",
      "13700000\n",
      "13800000\n",
      "13900000\n",
      "14000000\n",
      "14100000\n",
      "14200000\n",
      "14300000\n",
      "14400000\n",
      "14500000\n",
      "14600000\n",
      "14700000\n",
      "14800000\n",
      "14900000\n",
      "15000000\n",
      "15100000\n",
      "15200000\n",
      "15300000\n",
      "15400000\n",
      "15500000\n",
      "15600000\n",
      "15700000\n",
      "15800000\n",
      "15900000\n",
      "16000000\n",
      "16100000\n",
      "16200000\n",
      "16300000\n",
      "16400000\n",
      "16500000\n",
      "16600000\n",
      "16700000\n",
      "16800000\n",
      "16900000\n",
      "17000000\n",
      "17100000\n",
      "17200000\n",
      "17300000\n",
      "17400000\n",
      "17500000\n",
      "17600000\n",
      "17700000\n",
      "17800000\n",
      "17900000\n",
      "18000000\n",
      "18100000\n",
      "18200000\n",
      "18300000\n",
      "18400000\n",
      "18500000\n",
      "18600000\n",
      "18700000\n",
      "18800000\n",
      "18900000\n",
      "19000000\n",
      "19100000\n",
      "19200000\n",
      "19300000\n",
      "19400000\n",
      "19500000\n",
      "19600000\n",
      "19700000\n",
      "19800000\n",
      "19900000\n",
      "20000000\n",
      "20100000\n",
      "20200000\n",
      "20300000\n",
      "20400000\n",
      "20500000\n",
      "20600000\n",
      "20700000\n",
      "20800000\n",
      "20900000\n",
      "21000000\n",
      "21100000\n",
      "21200000\n",
      "21300000\n",
      "21400000\n",
      "21500000\n",
      "21600000\n",
      "21700000\n",
      "21800000\n",
      "21900000\n",
      "22000000\n",
      "22100000\n",
      "22200000\n",
      "22300000\n",
      "22400000\n",
      "22500000\n",
      "22600000\n",
      "22700000\n",
      "22800000\n",
      "22900000\n",
      "23000000\n",
      "23100000\n",
      "23200000\n",
      "23300000\n",
      "23400000\n",
      "23500000\n",
      "23600000\n",
      "23700000\n",
      "23800000\n",
      "23900000\n",
      "24000000\n",
      "24100000\n",
      "24200000\n",
      "24300000\n",
      "24400000\n",
      "24500000\n",
      "24600000\n",
      "24700000\n",
      "24800000\n",
      "24900000\n",
      "25000000\n",
      "25100000\n",
      "25200000\n",
      "25300000\n",
      "25400000\n",
      "25500000\n",
      "25600000\n",
      "25700000\n",
      "25800000\n",
      "25900000\n",
      "26000000\n",
      "26100000\n",
      "26200000\n",
      "26300000\n",
      "26400000\n",
      "26500000\n",
      "26600000\n",
      "26700000\n",
      "26800000\n",
      "26900000\n",
      "27000000\n",
      "27100000\n",
      "27200000\n",
      "27300000\n",
      "27400000\n",
      "27500000\n",
      "27600000\n",
      "27700000\n",
      "27800000\n",
      "27900000\n",
      "28000000\n",
      "28100000\n",
      "28200000\n",
      "28300000\n",
      "28400000\n",
      "28500000\n",
      "28600000\n",
      "28700000\n",
      "28800000\n",
      "28900000\n",
      "29000000\n",
      "29100000\n",
      "29200000\n",
      "29300000\n",
      "29400000\n",
      "29500000\n",
      "29600000\n",
      "29700000\n",
      "29800000\n",
      "29900000\n",
      "Data generation complete.\n"
     ]
    }
   ],
   "source": [
    "# create training data\n",
    "\n",
    "import numpy as np\n",
    "import scipy.signal as sig\n",
    "from numpy import linalg as LA\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create training and test data for the NN\n",
    "\n",
    "epochs = 30000\n",
    "batch_size = 1000\n",
    "\n",
    "# number of random channels that will be trained and tested on\n",
    "num_train=epochs*batch_size\n",
    "num_test=batch_size\n",
    "\n",
    "data_length = 20\n",
    "omega = 1/100 # rate of rotation, number of rotations / symbol\n",
    "\n",
    "\n",
    "# QPSK\n",
    "data_train = np.zeros((num_train, data_length,2))\n",
    "data_test = np.zeros((num_test, data_length,2))\n",
    "start_train = np.zeros((num_train, 2))\n",
    "start_test = np.zeros((num_test, 2))\n",
    "\n",
    "for i in range(num_train):\n",
    "    \n",
    "    # first choose a random starting point on the unit circle\n",
    "    # there are 2pi radians in a circle, choose random number between 0 and 2pi\n",
    "    random_start = np.random.rand()*2*np.pi\n",
    "#     random_start = 0\n",
    "    start_train[i,:] = np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    # real part of data\n",
    "    data_train[i,0,:]=np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_train[i,j,0]=(data_train[i,j-1,0]*np.cos(omega)-data_train[i,j-1,1]*np.sin(omega))\n",
    "        data_train[i,j,1]=(data_train[i,j-1,0]*np.sin(omega)+data_train[i,j-1,1]*np.cos(omega))\n",
    "        \n",
    "    if i % 100000 == 0:\n",
    "        print(i)\n",
    "    \n",
    "\n",
    "for i in range(num_test):\n",
    "    # first choose a random starting point on the unit circle\n",
    "    # there are 2pi radians in a circle, choose random number between 0 and 2pi\n",
    "    random_start = np.random.rand()*2*np.pi\n",
    "#     random_start = 0 \n",
    "    start_test[i,:]=np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    # real part of data\n",
    "    data_test[i,0,:]=np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_test[i,j,0]=(data_test[i,j-1,0]*np.cos(omega)-data_test[i,j-1,1]*np.sin(omega))\n",
    "        data_test[i,j,1]=(data_test[i,j-1,0]*np.sin(omega)+data_test[i,j-1,1]*np.cos(omega))\n",
    "\n",
    "print(\"Data generation complete.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.607843270062445\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'data_length' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-d5e0def41395>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# plot the movement\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'go'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'data_length' is not defined"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy.signal as sig\n",
    "from numpy import linalg as LA\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "random_start = np.random.rand()*2*np.pi\n",
    "print(random_start)\n",
    "\n",
    "np.array([np.cos(random_start), np.sin(random_start)])\n",
    "\n",
    "plt.figure()\n",
    "# plot unit circle\n",
    "t = np.linspace(0,np.pi*2,100)\n",
    "plt.plot(np.cos(t), np.sin(t), linewidth=1)\n",
    "\n",
    "k = 5\n",
    "\n",
    "# plot the movement\n",
    "for i in range(data_length):\n",
    "    plt.plot(data_train[k,i,0],data_train[k,i,1], 'go')\n",
    "\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "# pertrub weights by random number between -delta and delta\n",
    "delta = 0.9\n",
    "\n",
    "np.random.uniform(-delta,delta)\n",
    "\n",
    "w1 = ([[np.cos(omega)+np.random.uniform(-delta,delta),-np.sin(omega)+np.random.uniform(-delta,delta)],\n",
    "       [np.sin(omega)+np.random.uniform(-delta,delta), np.cos(omega)+np.random.uniform(-delta,delta)]])\n",
    "b1 = ([[np.random.uniform(-delta,delta),np.random.uniform(-delta,delta)]])\n",
    "\n",
    "w1_init = tf.constant_initializer(w1)\n",
    "b1_init = tf.constant_initializer(b1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import linalg as LA\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# fix random seed for reproducibility\n",
    "np.random.seed()\n",
    "\n",
    "learning_rate = 0.01\n",
    "decay = 1 - 10*1e-10\n",
    "\n",
    "state_size = 2 # state size\n",
    "output_size = 2 # size of the output\n",
    "\n",
    "batch_data_placeholder = tf.placeholder(tf.float32, [None, data_length,2])\n",
    "batch_start_placeholder = tf.placeholder(tf.float32, [None, 2])\n",
    "\n",
    "init_state = tf.placeholder(tf.float32, [None, state_size])\n",
    "\n",
    "# Unpack columns\n",
    "data = tf.unstack(batch_data_placeholder, axis=1)\n",
    "\n",
    "# Forward pass\n",
    "current_state = init_state\n",
    "est_data_series = []\n",
    "for current_data in data:\n",
    "    \n",
    "#     input_start_state_concatenated = tf.concat([batch_start_placeholder, current_state],1) \n",
    "    \n",
    "#     layer_1 = tf.layers.dense(\n",
    "#       current_state, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_2 = tf.layers.dense(\n",
    "#       layer_1, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_3 = tf.layers.dense(\n",
    "#       layer_2, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_4 = tf.layers.dense(\n",
    "#       layer_3, 100, tf.nn.sigmoid, use_bias=True)\n",
    "    \n",
    "    \n",
    "#     layer_5 = tf.layers.dense(\n",
    "#       layer_4, state_size, activation=tf.identity, use_bias=True)\n",
    "\n",
    "#     layer_1 = tf.layers.dense(\n",
    "#         current_state, output_size, activation=tf.identity, use_bias=True)\n",
    "    \n",
    "    layer_1 = tf.layers.dense(\n",
    "        current_state, output_size, activation=tf.identity, use_bias=True,\n",
    "        kernel_initializer=w1_init, bias_initializer=b1_init)\n",
    "\n",
    "#     layer_3 = tf.layers.dense(\n",
    "#         layer_1, state_size, activation=tf.identity, use_bias=True)\n",
    "\n",
    "    next_state = layer_1\n",
    "#     print(next_state[:,0])\n",
    "\n",
    "#     layer_4 = tf.layers.dense(\n",
    "#          layer_3, output_size, activation=tf.identity, use_bias=True)\n",
    "\n",
    "    est_data_series.append(next_state)\n",
    "#     print(est_data_series)\n",
    "#     print(np.array(est_data_series).shape)\n",
    "    current_state = next_state\n",
    "\n",
    "# est_data_series = (np.transpose(np.array(est_data_series), axes=(1, 0, 2))).tolist()    \n",
    "\n",
    "est_data_series=tf.transpose(est_data_series, perm=[1,0,2])\n",
    "# estimation_cost = tf.losses.mean_squared_error(data,est_data_series)\n",
    "squared_difference = tf.squared_difference(batch_data_placeholder, est_data_series)\n",
    "\n",
    "estimation_cost = tf.reduce_mean(tf.squared_difference(batch_data_placeholder, est_data_series))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(estimation_cost)\n",
    "                                                                                  \n",
    "# finally setup the initialisation operator\n",
    "init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Train Cost 10.074631690979004, Test Cost: 8.33143138885498\n",
      "Epoch 100, Train Cost 0.6247743368148804, Test Cost: 0.6011766791343689\n",
      "Epoch 200, Train Cost 0.4647752046585083, Test Cost: 0.46234267950057983\n",
      "Epoch 300, Train Cost 0.433489590883255, Test Cost: 0.4301551878452301\n",
      "Epoch 400, Train Cost 0.40760672092437744, Test Cost: 0.4066472053527832\n",
      "Epoch 500, Train Cost 0.3897833526134491, Test Cost: 0.38742566108703613\n",
      "Epoch 600, Train Cost 0.36804285645484924, Test Cost: 0.3667201101779938\n",
      "Epoch 700, Train Cost 0.3523634076118469, Test Cost: 0.3516514301300049\n",
      "Epoch 800, Train Cost 0.33673277497291565, Test Cost: 0.33620184659957886\n",
      "Epoch 900, Train Cost 0.31870782375335693, Test Cost: 0.3181261122226715\n",
      "Epoch 1000, Train Cost 0.2977922260761261, Test Cost: 0.2960728108882904\n",
      "Epoch 1100, Train Cost 0.276638925075531, Test Cost: 0.276141494512558\n",
      "Epoch 1200, Train Cost 0.25792112946510315, Test Cost: 0.2577212154865265\n",
      "Epoch 1300, Train Cost 0.24079199135303497, Test Cost: 0.24069365859031677\n",
      "Epoch 1400, Train Cost 0.22343920171260834, Test Cost: 0.22169692814350128\n",
      "Epoch 1500, Train Cost 0.20627565681934357, Test Cost: 0.20486247539520264\n",
      "Epoch 1600, Train Cost 0.19650258123874664, Test Cost: 0.19388948380947113\n",
      "Epoch 1700, Train Cost 0.18518659472465515, Test Cost: 0.18457180261611938\n",
      "Epoch 1800, Train Cost 0.1785726100206375, Test Cost: 0.17685849964618683\n",
      "Epoch 1900, Train Cost 0.171634703874588, Test Cost: 0.17109180986881256\n",
      "Epoch 2000, Train Cost 0.16439896821975708, Test Cost: 0.1638002246618271\n",
      "Epoch 2100, Train Cost 0.1582908183336258, Test Cost: 0.15758991241455078\n",
      "Epoch 2200, Train Cost 0.15107600390911102, Test Cost: 0.15156783163547516\n",
      "Epoch 2300, Train Cost 0.14542460441589355, Test Cost: 0.14572608470916748\n",
      "Epoch 2400, Train Cost 0.14019010961055756, Test Cost: 0.14034032821655273\n",
      "Epoch 2500, Train Cost 0.13654804229736328, Test Cost: 0.13565710186958313\n",
      "Epoch 2600, Train Cost 0.13263167440891266, Test Cost: 0.13076665997505188\n",
      "Epoch 2700, Train Cost 0.12747341394424438, Test Cost: 0.12629087269306183\n",
      "Epoch 2800, Train Cost 0.12259121239185333, Test Cost: 0.12165921926498413\n",
      "Epoch 2900, Train Cost 0.11708337068557739, Test Cost: 0.11678365617990494\n",
      "Epoch 3000, Train Cost 0.11183413118124008, Test Cost: 0.1114143431186676\n",
      "Epoch 3100, Train Cost 0.10586249828338623, Test Cost: 0.10531077533960342\n",
      "Epoch 3200, Train Cost 0.0982903316617012, Test Cost: 0.09829078614711761\n",
      "Epoch 3300, Train Cost 0.09234631061553955, Test Cost: 0.09018079936504364\n",
      "Epoch 3400, Train Cost 0.08206828683614731, Test Cost: 0.08095555752515793\n",
      "Epoch 3500, Train Cost 0.07627314329147339, Test Cost: 0.07468514889478683\n",
      "Epoch 3600, Train Cost 0.06958148628473282, Test Cost: 0.06972385197877884\n",
      "Epoch 3700, Train Cost 0.06530480086803436, Test Cost: 0.06478288024663925\n",
      "Epoch 3800, Train Cost 0.06050019711256027, Test Cost: 0.05957028269767761\n",
      "Epoch 3900, Train Cost 0.05446944758296013, Test Cost: 0.05398872122168541\n",
      "Epoch 4000, Train Cost 0.04967596009373665, Test Cost: 0.04850705713033676\n",
      "Epoch 4100, Train Cost 0.04503164067864418, Test Cost: 0.04329115152359009\n",
      "Epoch 4200, Train Cost 0.03795513138175011, Test Cost: 0.036865994334220886\n",
      "Epoch 4300, Train Cost 0.028138890862464905, Test Cost: 0.027871225029230118\n",
      "Epoch 4400, Train Cost 0.02266683429479599, Test Cost: 0.022340161725878716\n",
      "Epoch 4500, Train Cost 0.017810074612498283, Test Cost: 0.017824362963438034\n",
      "Epoch 4600, Train Cost 0.01437492948025465, Test Cost: 0.014128821901977062\n",
      "Epoch 4700, Train Cost 0.012033586390316486, Test Cost: 0.011700941249728203\n",
      "Epoch 4800, Train Cost 0.010679871775209904, Test Cost: 0.010181697085499763\n",
      "Epoch 4900, Train Cost 0.009271877817809582, Test Cost: 0.009093278087675571\n",
      "Epoch 5000, Train Cost 0.008219781331717968, Test Cost: 0.008139721117913723\n",
      "Epoch 5100, Train Cost 0.007465565577149391, Test Cost: 0.007204487454146147\n",
      "Epoch 5200, Train Cost 0.006469954736530781, Test Cost: 0.006173703353852034\n",
      "Epoch 5300, Train Cost 0.005114307161420584, Test Cost: 0.004888586234301329\n",
      "Epoch 5400, Train Cost 0.00325645599514246, Test Cost: 0.0030625255312770605\n",
      "Epoch 5500, Train Cost 0.0009093136759474874, Test Cost: 0.0008524048025719821\n",
      "Epoch 5600, Train Cost 0.00010857723100343719, Test Cost: 0.00011021724640158936\n",
      "Epoch 5700, Train Cost 2.26224037760403e-05, Test Cost: 2.2906177036929876e-05\n",
      "Epoch 5800, Train Cost 5.17537273481139e-06, Test Cost: 5.152289304533042e-06\n",
      "Epoch 5900, Train Cost 1.1028504331989097e-06, Test Cost: 1.1371163282092311e-06\n",
      "Epoch 6000, Train Cost 2.290043710218015e-07, Test Cost: 2.387913582424517e-07\n",
      "Epoch 6100, Train Cost 4.7289116622550864e-08, Test Cost: 4.664300234935581e-08\n",
      "Epoch 6200, Train Cost 8.237539539379668e-09, Test Cost: 8.379767990618348e-09\n",
      "Epoch 6300, Train Cost 1.3792497144393678e-09, Test Cost: 1.3878643789766443e-09\n",
      "Epoch 6400, Train Cost 2.142748201094946e-10, Test Cost: 2.092439832512838e-10\n",
      "Epoch 6500, Train Cost 2.941226082131543e-11, Test Cost: 2.8475107688441526e-11\n",
      "Epoch 6600, Train Cost 3.782992148704256e-12, Test Cost: 3.6890161064784266e-12\n",
      "Epoch 6700, Train Cost 5.937462435774699e-13, Test Cost: 5.792236265275696e-13\n",
      "Epoch 6800, Train Cost 2.109334435633839e-13, Test Cost: 2.016665507547158e-13\n",
      "Epoch 6900, Train Cost 1.4388545964503369e-13, Test Cost: 1.5181207528060237e-13\n",
      "Epoch 7000, Train Cost 1.654366883286143e-13, Test Cost: 1.5046614670367886e-13\n",
      "Epoch 7100, Train Cost 1.4737667195559995e-13, Test Cost: 1.5862879315219763e-13\n",
      "Epoch 7200, Train Cost 1.7165998440413222e-12, Test Cost: 2.149913493759703e-12\n",
      "Epoch 7300, Train Cost 8.807666063148645e-08, Test Cost: 1.1794287502198131e-07\n",
      "Epoch 7400, Train Cost 1.4146093008093885e-06, Test Cost: 1.4102574823482428e-06\n",
      "Epoch 7500, Train Cost 1.850657156410307e-07, Test Cost: 2.2347678907408408e-07\n",
      "Epoch 7600, Train Cost 1.3956116617919179e-06, Test Cost: 5.775399927188118e-07\n",
      "Epoch 7700, Train Cost 7.943218349737435e-08, Test Cost: 8.339797119560899e-08\n",
      "Epoch 7800, Train Cost 8.144148750943714e-07, Test Cost: 4.775069442075619e-07\n",
      "Epoch 7900, Train Cost 1.381402944389265e-05, Test Cost: 1.209085985465208e-05\n",
      "Epoch 8000, Train Cost 8.641594035907474e-07, Test Cost: 9.253645316675829e-07\n",
      "Epoch 8100, Train Cost 3.774971901293611e-06, Test Cost: 4.824945790460333e-06\n",
      "Epoch 8200, Train Cost 1.8392765923636034e-05, Test Cost: 2.0770896298927255e-05\n",
      "Epoch 8300, Train Cost 6.811977982579265e-06, Test Cost: 1.3306814707902959e-06\n",
      "Epoch 8400, Train Cost 1.958711800398305e-05, Test Cost: 1.7273781850235537e-05\n",
      "Epoch 8500, Train Cost 1.3961027889308752e-06, Test Cost: 1.6237985391853726e-06\n",
      "Epoch 8600, Train Cost 3.500590173644014e-05, Test Cost: 6.568332901224494e-05\n",
      "Epoch 8700, Train Cost 1.2389299627102446e-05, Test Cost: 1.1473321137600578e-05\n",
      "Epoch 8800, Train Cost 0.00011659114534268156, Test Cost: 8.235870336648077e-05\n",
      "Epoch 8900, Train Cost 4.183908913546475e-06, Test Cost: 4.0886789065552875e-06\n",
      "Epoch 9000, Train Cost 1.2469549801608082e-06, Test Cost: 1.0778680916700978e-06\n",
      "Epoch 9100, Train Cost 2.4053679226199165e-06, Test Cost: 3.1986255635274574e-06\n",
      "Epoch 9200, Train Cost 4.500597697187914e-06, Test Cost: 4.797166639036732e-06\n",
      "Epoch 9300, Train Cost 1.4337435459310655e-05, Test Cost: 2.6191490178462118e-05\n",
      "Epoch 9400, Train Cost 1.681954358900839e-06, Test Cost: 1.4582102494387073e-06\n",
      "Epoch 9500, Train Cost 2.6132196580874734e-05, Test Cost: 2.2151405573822558e-05\n",
      "Epoch 9600, Train Cost 2.051689023119252e-07, Test Cost: 2.629300297485315e-07\n",
      "Epoch 9700, Train Cost 7.961387018440291e-05, Test Cost: 0.00011249481758568436\n",
      "Epoch 9800, Train Cost 4.0682908775124815e-07, Test Cost: 4.4833203105554276e-07\n",
      "Epoch 9900, Train Cost 1.1832119525934104e-05, Test Cost: 1.4807456864218693e-05\n",
      "Epoch 10000, Train Cost 1.2392852113407571e-05, Test Cost: 8.97665540833259e-06\n",
      "Epoch 10100, Train Cost 0.00012353414786048234, Test Cost: 4.577037543640472e-05\n",
      "Epoch 10200, Train Cost 8.659815193823306e-07, Test Cost: 4.282285885892634e-07\n",
      "Epoch 10300, Train Cost 1.832545967772603e-05, Test Cost: 2.924277760030236e-05\n",
      "Epoch 10400, Train Cost 2.73382192972349e-05, Test Cost: 3.055029810639098e-05\n",
      "Epoch 10500, Train Cost 2.4352104446734302e-05, Test Cost: 2.5751680368557572e-05\n",
      "Epoch 10600, Train Cost 2.1796384430672333e-07, Test Cost: 2.3840603091684898e-07\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10700, Train Cost 2.7096024496131577e-05, Test Cost: 1.5965086276992224e-05\n",
      "Epoch 10800, Train Cost 3.936021312256344e-05, Test Cost: 5.943515134276822e-05\n",
      "Epoch 10900, Train Cost 8.520310501580752e-08, Test Cost: 8.991048616735497e-08\n",
      "Epoch 11000, Train Cost 1.2613890021384577e-06, Test Cost: 1.207048399010091e-06\n",
      "Epoch 11100, Train Cost 1.0428115274407901e-05, Test Cost: 7.647117854503449e-06\n",
      "Epoch 11200, Train Cost 7.925749741843902e-06, Test Cost: 2.5509109491395066e-06\n",
      "Epoch 11300, Train Cost 5.823619630973553e-06, Test Cost: 5.158398835192202e-06\n",
      "Epoch 11400, Train Cost 2.3605043679708615e-05, Test Cost: 2.050530383712612e-05\n",
      "Epoch 11500, Train Cost 7.239072874654084e-05, Test Cost: 6.69220244162716e-05\n",
      "Epoch 11600, Train Cost 1.0802194339021298e-07, Test Cost: 2.811501360611146e-07\n",
      "Epoch 11700, Train Cost 0.00013788300566375256, Test Cost: 7.465656381100416e-05\n",
      "Epoch 11800, Train Cost 1.8317804233447532e-06, Test Cost: 2.4967175704659894e-06\n",
      "Epoch 11900, Train Cost 6.109257100206378e-09, Test Cost: 5.502019284620019e-09\n",
      "Epoch 12000, Train Cost 2.0989295080653392e-05, Test Cost: 2.4210556148318574e-05\n",
      "Epoch 12100, Train Cost 3.452073116250176e-08, Test Cost: 1.72028471467911e-08\n",
      "Epoch 12200, Train Cost 2.453772867738735e-05, Test Cost: 1.8390248442301527e-05\n",
      "Epoch 12300, Train Cost 2.6864645406021737e-05, Test Cost: 2.6757023078971542e-05\n",
      "Epoch 12400, Train Cost 2.4457380277453922e-05, Test Cost: 2.6686846467782743e-05\n",
      "Epoch 12500, Train Cost 4.524048563325778e-06, Test Cost: 5.4381348491006065e-06\n",
      "Epoch 12600, Train Cost 1.1170452125952579e-05, Test Cost: 1.0649840078258421e-05\n",
      "Epoch 12700, Train Cost 9.414664532414463e-07, Test Cost: 1.2831710591854062e-06\n",
      "Epoch 12800, Train Cost 2.9391563657554798e-05, Test Cost: 1.956258711288683e-05\n",
      "Epoch 12900, Train Cost 6.504466796286579e-07, Test Cost: 4.305491358991276e-07\n",
      "Epoch 13000, Train Cost 1.0251407729811035e-05, Test Cost: 1.0006047887145542e-05\n",
      "Epoch 13100, Train Cost 1.1528666618687566e-05, Test Cost: 7.1717886385158636e-06\n",
      "Epoch 13200, Train Cost 6.51728305456345e-06, Test Cost: 1.349298509012442e-05\n",
      "Epoch 13300, Train Cost 5.235478965914808e-06, Test Cost: 5.297966254147468e-06\n",
      "Epoch 13400, Train Cost 3.401007404590928e-07, Test Cost: 3.12641901700772e-07\n",
      "Epoch 13500, Train Cost 1.0808293149366932e-09, Test Cost: 7.717760319714273e-10\n",
      "Epoch 13600, Train Cost 0.000107945132185705, Test Cost: 9.008227789308876e-05\n",
      "Epoch 13700, Train Cost 5.8299751799495425e-06, Test Cost: 4.793338121089619e-06\n",
      "Epoch 13800, Train Cost 2.0385363313835114e-05, Test Cost: 2.7279278583591804e-05\n",
      "Epoch 13900, Train Cost 8.958396392699797e-06, Test Cost: 1.1428314792283345e-05\n",
      "Epoch 14000, Train Cost 9.560844773659483e-05, Test Cost: 5.005118146073073e-05\n",
      "Epoch 14100, Train Cost 1.4572642612620257e-05, Test Cost: 1.4807596926402766e-05\n",
      "Epoch 14200, Train Cost 7.630378604517318e-07, Test Cost: 9.058915111381793e-07\n",
      "Epoch 14300, Train Cost 2.859944061128772e-06, Test Cost: 6.801250833632366e-07\n",
      "Epoch 14400, Train Cost 3.215964852643083e-06, Test Cost: 6.9970087679394055e-06\n",
      "Epoch 14500, Train Cost 7.843992534617428e-06, Test Cost: 8.845666343404446e-06\n",
      "Epoch 14600, Train Cost 2.0271649191272445e-05, Test Cost: 4.418787648319267e-05\n",
      "Epoch 14700, Train Cost 9.241128282155842e-05, Test Cost: 9.801294072531164e-05\n",
      "Epoch 14800, Train Cost 2.798768036882393e-06, Test Cost: 1.5727248410257744e-06\n",
      "Epoch 14900, Train Cost 1.8511016605771147e-05, Test Cost: 2.60745200648671e-05\n",
      "Epoch 15000, Train Cost 5.3267099247023e-06, Test Cost: 7.938377166283317e-06\n",
      "Epoch 15100, Train Cost 2.0244344341335818e-05, Test Cost: 2.7399324608268216e-05\n",
      "Epoch 15200, Train Cost 6.024516551406123e-06, Test Cost: 8.530683771823533e-06\n",
      "Epoch 15300, Train Cost 2.3030843294691294e-05, Test Cost: 3.0483481168630533e-05\n",
      "Epoch 15400, Train Cost 1.3982216842123307e-06, Test Cost: 1.1806030215666397e-06\n",
      "Epoch 15500, Train Cost 0.00011576825636439025, Test Cost: 0.0001403400965500623\n",
      "Epoch 15600, Train Cost 2.741480420809239e-05, Test Cost: 2.42102105403319e-05\n",
      "Epoch 15700, Train Cost 8.039995009312406e-05, Test Cost: 7.60142647777684e-05\n",
      "Epoch 15800, Train Cost 9.234105164068751e-06, Test Cost: 6.0776883401558734e-06\n",
      "Epoch 15900, Train Cost 9.805456102185417e-06, Test Cost: 1.4517223462462425e-05\n",
      "Epoch 16000, Train Cost 7.946965524752159e-06, Test Cost: 5.898363269807305e-06\n",
      "Epoch 16100, Train Cost 3.356545130372979e-05, Test Cost: 4.84416086692363e-05\n",
      "Epoch 16200, Train Cost 1.3767265549802232e-08, Test Cost: 2.5767945999177755e-08\n",
      "Epoch 16300, Train Cost 1.038274605180689e-10, Test Cost: 8.059180406361222e-11\n",
      "Epoch 16400, Train Cost 8.345887181349099e-06, Test Cost: 7.133482540666591e-06\n",
      "Epoch 16500, Train Cost 3.609668510762276e-06, Test Cost: 2.8691499665001174e-06\n",
      "Epoch 16600, Train Cost 1.8240524468637886e-06, Test Cost: 1.2017469998681918e-05\n",
      "Epoch 16700, Train Cost 7.298144311107535e-09, Test Cost: 6.974889554811625e-09\n",
      "Epoch 16800, Train Cost 1.0916050086962059e-05, Test Cost: 1.0388946975581348e-05\n",
      "Epoch 16900, Train Cost 7.496762555092573e-05, Test Cost: 0.0001224331499543041\n",
      "Epoch 17000, Train Cost 9.292333561461419e-05, Test Cost: 0.00016350296209566295\n",
      "Epoch 17100, Train Cost 3.6050440854751287e-08, Test Cost: 2.2693662060646602e-08\n",
      "Epoch 17200, Train Cost 0.00017560790001880378, Test Cost: 0.00032087284489534795\n",
      "Epoch 17300, Train Cost 1.2987258912744437e-07, Test Cost: 7.929418899266238e-08\n",
      "Epoch 17400, Train Cost 1.2955450601204177e-10, Test Cost: 8.986514310471705e-11\n",
      "Epoch 17500, Train Cost 1.5078314845595742e-06, Test Cost: 1.9020443460249226e-06\n",
      "Epoch 17600, Train Cost 4.523815368884243e-05, Test Cost: 5.8700894442154095e-05\n",
      "Epoch 17700, Train Cost 4.2494043555052485e-06, Test Cost: 3.6887656733597396e-06\n",
      "Epoch 17800, Train Cost 5.2477134886430576e-05, Test Cost: 8.285080548375845e-05\n",
      "Epoch 17900, Train Cost 1.599053939571604e-05, Test Cost: 2.1735277186962776e-05\n",
      "Epoch 18000, Train Cost 2.5640600142651238e-05, Test Cost: 1.806104410206899e-05\n",
      "Epoch 18100, Train Cost 4.925024040858261e-05, Test Cost: 4.736816481454298e-05\n",
      "Epoch 18200, Train Cost 2.356120057811495e-05, Test Cost: 4.399797398946248e-05\n",
      "Epoch 18300, Train Cost 7.008404281805269e-06, Test Cost: 4.6280993046821095e-06\n",
      "Epoch 18400, Train Cost 2.0522300474112853e-05, Test Cost: 7.928749255370349e-05\n",
      "Epoch 18500, Train Cost 1.2895166889848042e-07, Test Cost: 1.2080262479230441e-07\n",
      "Epoch 18600, Train Cost 2.5634329728352334e-11, Test Cost: 3.419351260469661e-11\n",
      "Epoch 18700, Train Cost 1.7139158444479108e-05, Test Cost: 1.518019416835159e-05\n",
      "Epoch 18800, Train Cost 1.966598688341037e-07, Test Cost: 3.968945350152353e-07\n",
      "Epoch 18900, Train Cost 1.8504109903005883e-05, Test Cost: 5.5988639360293746e-05\n",
      "Epoch 19000, Train Cost 1.3563414995587664e-06, Test Cost: 1.701206315374293e-06\n",
      "Epoch 19100, Train Cost 2.923524107245612e-07, Test Cost: 3.127929630863946e-07\n",
      "Epoch 19200, Train Cost 2.8288870453252457e-05, Test Cost: 5.457680890685879e-05\n",
      "Epoch 19300, Train Cost 3.432311359574669e-07, Test Cost: 2.674491952348035e-07\n",
      "Epoch 19400, Train Cost 4.8063157009892166e-05, Test Cost: 4.468079350772314e-05\n",
      "Epoch 19500, Train Cost 3.5966259019915015e-05, Test Cost: 3.548589666024782e-05\n",
      "Epoch 19600, Train Cost 7.928304694360122e-05, Test Cost: 4.557894862955436e-05\n",
      "Epoch 19700, Train Cost 4.490727951633744e-05, Test Cost: 3.2013547752285376e-05\n",
      "Epoch 19800, Train Cost 1.0120551223735674e-06, Test Cost: 1.1823298109447933e-06\n",
      "Epoch 19900, Train Cost 7.188159088400425e-06, Test Cost: 6.745505288563436e-06\n",
      "Epoch 20000, Train Cost 3.202113657607697e-05, Test Cost: 1.542107929708436e-05\n",
      "Epoch 20100, Train Cost 6.264659987209598e-06, Test Cost: 8.040014108701143e-06\n",
      "Epoch 20200, Train Cost 8.124076884996612e-06, Test Cost: 7.645377991138957e-06\n",
      "Epoch 20300, Train Cost 8.126228203764185e-05, Test Cost: 2.8361087970552035e-05\n",
      "Epoch 20400, Train Cost 2.210527327406453e-06, Test Cost: 3.0319238248921465e-06\n",
      "Epoch 20500, Train Cost 0.00010069982090499252, Test Cost: 4.610400355886668e-05\n",
      "Epoch 20600, Train Cost 4.763056494994089e-05, Test Cost: 3.866958650178276e-05\n",
      "Epoch 20700, Train Cost 7.870607078075409e-05, Test Cost: 7.00685559422709e-05\n",
      "Epoch 20800, Train Cost 8.483604688080959e-06, Test Cost: 9.920664524543099e-06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20900, Train Cost 1.9403153146413388e-06, Test Cost: 2.661334292497486e-06\n",
      "Epoch 21000, Train Cost 2.099726088999887e-06, Test Cost: 1.3258006674732314e-06\n",
      "Epoch 21100, Train Cost 5.2797608987020794e-06, Test Cost: 4.8626870920998044e-06\n",
      "Epoch 21200, Train Cost 2.5482819182798266e-05, Test Cost: 2.647897599672433e-05\n",
      "Epoch 21300, Train Cost 1.2209777196403593e-05, Test Cost: 2.318796396139078e-05\n",
      "Epoch 21400, Train Cost 6.816835957579315e-05, Test Cost: 7.32640692149289e-05\n",
      "Epoch 21500, Train Cost 1.3869644135411363e-06, Test Cost: 2.8880040190415457e-06\n",
      "Epoch 21600, Train Cost 6.381455932569224e-06, Test Cost: 1.108344440581277e-05\n",
      "Epoch 21700, Train Cost 0.00012590225378517061, Test Cost: 0.00015939702279865742\n",
      "Epoch 21800, Train Cost 2.062298881355673e-05, Test Cost: 1.575743044668343e-05\n",
      "Epoch 21900, Train Cost 2.3170963459051563e-07, Test Cost: 5.433775527308171e-07\n",
      "Epoch 22000, Train Cost 3.8683003367623314e-05, Test Cost: 1.3938303709437605e-05\n",
      "Epoch 22100, Train Cost 6.018346425662457e-07, Test Cost: 6.123424896031793e-07\n",
      "Epoch 22200, Train Cost 2.090801535814535e-05, Test Cost: 1.9867537048412487e-05\n",
      "Epoch 22300, Train Cost 2.2234389973618818e-07, Test Cost: 2.0004327438982727e-07\n",
      "Epoch 22400, Train Cost 5.7702989579411224e-05, Test Cost: 1.4622095477534458e-05\n",
      "Epoch 22500, Train Cost 1.628020800126251e-05, Test Cost: 2.5346607799292542e-05\n",
      "Epoch 22600, Train Cost 6.375747148013033e-07, Test Cost: 1.4227916835807264e-06\n",
      "Epoch 22700, Train Cost 6.972065602894872e-05, Test Cost: 6.831013888586313e-05\n",
      "Epoch 22800, Train Cost 1.5950570286804577e-07, Test Cost: 2.847256723725877e-07\n",
      "Epoch 22900, Train Cost 3.597751856432296e-05, Test Cost: 3.116212610621005e-05\n",
      "Epoch 23000, Train Cost 1.986159148259503e-08, Test Cost: 2.8232273407979847e-08\n",
      "Epoch 23100, Train Cost 3.128837488475256e-05, Test Cost: 3.77212927560322e-05\n",
      "Epoch 23200, Train Cost 3.0133689961076016e-06, Test Cost: 2.72902025244548e-06\n",
      "Epoch 23300, Train Cost 4.654325493902434e-06, Test Cost: 6.721918452967657e-06\n",
      "Epoch 23400, Train Cost 9.761782848727307e-07, Test Cost: 7.454215733559977e-07\n",
      "Epoch 23500, Train Cost 2.97002316074213e-05, Test Cost: 1.5300129234674387e-05\n",
      "Epoch 23600, Train Cost 9.205697892866738e-07, Test Cost: 1.618252554180799e-06\n",
      "Epoch 23700, Train Cost 1.2817207561965915e-07, Test Cost: 1.3903826356909121e-07\n",
      "Epoch 23800, Train Cost 1.7555059912410798e-06, Test Cost: 1.1714172387655708e-06\n",
      "Epoch 23900, Train Cost 5.698004770238185e-06, Test Cost: 3.840814315481111e-06\n",
      "Epoch 24000, Train Cost 3.35938784701284e-06, Test Cost: 2.1210553313721903e-06\n",
      "Epoch 24100, Train Cost 1.5599642210872844e-05, Test Cost: 1.403644091624301e-05\n",
      "Epoch 24200, Train Cost 4.175284630036913e-05, Test Cost: 4.539890505839139e-05\n",
      "Epoch 24300, Train Cost 7.478247425751761e-06, Test Cost: 8.027874173421878e-06\n",
      "Epoch 24400, Train Cost 1.3663548088516109e-05, Test Cost: 4.997770156478509e-05\n",
      "Epoch 24500, Train Cost 2.4806097371765645e-06, Test Cost: 3.3312755931547144e-06\n",
      "Epoch 24600, Train Cost 2.946734412034857e-06, Test Cost: 2.2719859771314077e-06\n",
      "Epoch 24700, Train Cost 1.7814614693634212e-05, Test Cost: 1.0978264072036836e-05\n",
      "Epoch 24800, Train Cost 7.258808523147309e-07, Test Cost: 2.9502896268240875e-06\n",
      "Epoch 24900, Train Cost 7.028754112070601e-08, Test Cost: 5.192003271758949e-08\n",
      "Epoch 25000, Train Cost 1.3289466551213991e-05, Test Cost: 7.362682936218334e-06\n",
      "Epoch 25100, Train Cost 3.6488949263002723e-05, Test Cost: 5.150551805854775e-05\n",
      "Epoch 25200, Train Cost 1.0103075283041107e-06, Test Cost: 7.091397264957777e-07\n",
      "Epoch 25300, Train Cost 1.1660485142783727e-05, Test Cost: 9.178083018923644e-06\n",
      "Epoch 25400, Train Cost 4.5775999751640484e-05, Test Cost: 5.5451691878261045e-05\n",
      "Epoch 25500, Train Cost 2.118393283012665e-08, Test Cost: 1.4147814475506948e-08\n",
      "Epoch 25600, Train Cost 7.886215826147236e-06, Test Cost: 5.234977834334131e-06\n",
      "Epoch 25700, Train Cost 2.4350572402909165e-06, Test Cost: 1.8362258060733438e-06\n",
      "Epoch 25800, Train Cost 1.6747072777434369e-06, Test Cost: 5.148436230228981e-06\n",
      "Epoch 25900, Train Cost 2.7243186195846647e-06, Test Cost: 5.466676157084294e-06\n",
      "Epoch 26000, Train Cost 3.1820447475183755e-05, Test Cost: 1.3215169019531459e-05\n",
      "Epoch 26100, Train Cost 1.9543134044397448e-07, Test Cost: 2.658452729065175e-07\n",
      "Epoch 26200, Train Cost 3.740975898836041e-06, Test Cost: 2.1384241790656233e-06\n",
      "Epoch 26300, Train Cost 1.0805561942106578e-05, Test Cost: 9.564345418766607e-06\n",
      "Epoch 26400, Train Cost 4.99728969316493e-07, Test Cost: 5.338462756299123e-07\n",
      "Epoch 26500, Train Cost 2.2205718778423034e-05, Test Cost: 4.7021945647429675e-05\n",
      "Epoch 26600, Train Cost 2.0510472040768946e-06, Test Cost: 4.157076546107419e-06\n",
      "Epoch 26700, Train Cost 1.4658535008038598e-07, Test Cost: 2.2637090069110855e-07\n",
      "Epoch 26800, Train Cost 0.00011877524957526475, Test Cost: 8.750426059123129e-05\n",
      "Epoch 26900, Train Cost 4.286445687284868e-07, Test Cost: 3.6567007555277087e-07\n",
      "Epoch 27000, Train Cost 4.4095915541220165e-07, Test Cost: 3.30475103282879e-07\n",
      "Epoch 27100, Train Cost 2.3041616259433795e-06, Test Cost: 2.6901670935330912e-06\n",
      "Epoch 27200, Train Cost 4.221061772113899e-06, Test Cost: 1.3004249012738e-05\n",
      "Epoch 27300, Train Cost 1.0690372675981052e-07, Test Cost: 1.352765082174301e-07\n",
      "Epoch 27400, Train Cost 2.8819265935453586e-05, Test Cost: 3.791274502873421e-05\n",
      "Epoch 27500, Train Cost 1.8772417433865485e-06, Test Cost: 2.066994738925132e-06\n",
      "Epoch 27600, Train Cost 6.152927198854741e-06, Test Cost: 3.973331331508234e-06\n",
      "Epoch 27700, Train Cost 9.631309694668744e-06, Test Cost: 1.3831047908752225e-05\n",
      "Epoch 27800, Train Cost 7.150732017180417e-06, Test Cost: 6.087416295486037e-06\n",
      "Epoch 27900, Train Cost 1.49098013935145e-07, Test Cost: 2.1043018705313443e-07\n",
      "Epoch 28000, Train Cost 3.181360625603702e-06, Test Cost: 6.12050052950508e-06\n",
      "Epoch 28100, Train Cost 3.138304464300745e-06, Test Cost: 2.844455366357579e-06\n",
      "Epoch 28200, Train Cost 9.703719842946157e-05, Test Cost: 9.100223542191088e-05\n",
      "Epoch 28300, Train Cost 4.690462901635328e-06, Test Cost: 4.10774782722001e-06\n",
      "Epoch 28400, Train Cost 3.979007033194648e-06, Test Cost: 4.598130090016639e-06\n",
      "Epoch 28500, Train Cost 2.6222187443636358e-05, Test Cost: 3.63082617695909e-05\n",
      "Epoch 28600, Train Cost 3.387495553397457e-06, Test Cost: 4.448199888429372e-06\n",
      "Epoch 28700, Train Cost 6.789963435949176e-08, Test Cost: 1.1393343868348893e-07\n",
      "Epoch 28800, Train Cost 1.4421191735891625e-05, Test Cost: 8.462186087854207e-06\n",
      "Epoch 28900, Train Cost 1.1771898122958646e-08, Test Cost: 4.517846541318704e-09\n",
      "Epoch 29000, Train Cost 8.917157174437307e-06, Test Cost: 7.237945283122826e-06\n",
      "Epoch 29100, Train Cost 5.011384928366169e-06, Test Cost: 5.343085376807721e-06\n",
      "Epoch 29200, Train Cost 8.435494964942336e-05, Test Cost: 6.10613205935806e-05\n",
      "Epoch 29300, Train Cost 9.645008276493172e-07, Test Cost: 8.835374387672346e-07\n",
      "Epoch 29400, Train Cost 9.96066810330376e-05, Test Cost: 0.00011503298446768895\n",
      "Epoch 29500, Train Cost 1.2589746347657638e-06, Test Cost: 1.7226725503860507e-06\n",
      "Epoch 29600, Train Cost 1.949357829289511e-05, Test Cost: 2.97857968689641e-05\n",
      "Epoch 29700, Train Cost 1.9647359295049682e-05, Test Cost: 2.6949519451591186e-05\n",
      "Epoch 29800, Train Cost 0.00017271889373660088, Test Cost: 0.00019254985090810806\n",
      "Epoch 29900, Train Cost 5.33532329427544e-07, Test Cost: 5.116747274769295e-07\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztvX2cXWV57/29MpngJIEok/gKk/gSrEIENaW19Nj2wVYJRSQ9onWSptjnCQG1wR6P1Y4VaZunij2ncIoJ5njgRDIVqAQokhxecopvR2sDAoFaBDEJCKKZaCAkJSG5zh97rXHNnvW699p7rz379/189mf2rLX2Wte919r3dd/X223ujhBCCJHEtE4LIIQQotpIUQghhEhFikIIIUQqUhRCCCFSkaIQQgiRihSFEEKIVKQohMiBmbmZvSZ4f6WZ/XmnZeoEZjZkZvvMrK/Tsoj2IUUhWoKZ7TCzp8xsVmTb/2tmd0X+dzPbbmbTItv+ysz+Z8I5f9PMjgQdVfT1lla2pR53X+Xuf9nOa1YFd9/l7rPd/XCnZRHtQ4pCtJLpwOqMY14OvLfAOZ8IOqro61uNi1gNNEIXVUaKQrSSzwIfMbMXphxzKXCJmU1v9mJm9koz+6qZPWNmd5jZFWa2Mdj3m2b2eN3xO8zsbcH7U83sW2b2czN7MvjsjITr/E8z+6vg/S11s5sjZvaHwb5fCuTYY2YPmdm5dedYZ2abzexZ4LdytO8uM/tLM/tm0MbbzWxuZP87zezBoA13mdnrUs7lZrbKzB42s5+Z2efMzIJ908zsE2a208x+YmZfNLM5wb4FwWenB///oZk9GsjzQzMbjlzj/Wb2veD8t5nZ/Kw2imoiRSFayTbgLuAjKcdsAp4G/rCE6/09cDcwF/hLYEWBzx4GPhx89i3A6cCFWR9y97PCmQ3wH4EfA1sDk9sdgUwvBn4fWGtmJ0Y+/j5gDXA08A0ze5+Z3Z9xyfcB5wXnnEHw3ZrZCcCXgIuAecBm4JYkZRfwu8AvAycD5wJvD7b/YfD6LeBVwGzgivoPB238b8AZ7n408GvAvcG+dwF/BiwN5Pl6IJ/oQqQoRKv5JPAhM5uXsN+BPwc+aWZH5Tjfy4MRc/Q1y8yGqHV6f+7uz7n714Bb8grp7ne7+7fd/Xl33wF8HviNvJ8POuovAu9x98eodcI73P3q4Jz3ADdQUyYhN7v7N939iLv/u7v/vbu/IeNSV7v79939AHA9cEqw/T3Are5+h7sfAv4GGKDWeSfxaXf/ubvvAv4pcq5h4L+6+6Puvg/4OPDehFnfEeAkMxtw9yfd/cFg+/nAX7v799z9eeD/B07RrKI7kaIQLcXdHwC+Anws5ZjNwC5gZY5TPuHuL6x7PUvN1/Gz4H3IzrxymtkJZvYVM/uxmT1NrWObm/W54LNzgJupKamvB5vnA78SVWjUOuCXRj76WF75Ivw48n4/tdE+1No/3l53PxKc/xXNnit4Px14SfTDwXf9HmAV8KSZ3WpmvxTsng9cHmn7HsAy5BEVRYpCtIOLgf+P9E7iE8AIMLPBazwJvCgaZQUMRd4/Gz134DyOznLWAf8GLHT3Y6iZTSzrokHE1t8D/+Tun4/segz4ap1Cm+3uF0SOKbN08xPUOudQLgOOB37U7LmofY/PA0/VH+jut7n7bwMvo/b9/fdg12PA+XXtH3D3/9OAPKLDSFGIluPujwDXAX+ccsxdwHaK+RWin99JzSdyiZnNMLNfB86KHPJ94AVmdqaZ9VNTTFFT19HUfCX7glFxtENPYw0wi8nRXV8BTjCz5WbWH7x+Oc3B3CTXA2ea2elB+/4T8BzQSMf8JeDDQXDAbGqzq+sCE9I4ZvaSwIE+K7jWPmq+HoArgY+HPhkzm2Nm726oZaLjSFGIdvEX1DrUND4BHJtxzMttch7F7wX73gf8CjUzx8XUfAYAuPteas7pL1AbZT8LRKOgPhJ8/hlqo+LrcrWq5qT+VeBnEXmG3f0Z4Heohf4+Qc3M8xkmKqcJmNmwmT2YtD8Nd38IWAb8HbCbmpI8y90PNnC6q4BrgK8BPwT+HfhQzHHTqCmkJ6h9579BEADg7jdSa++1gSnvAeCMBmQRFcC0cJGYqpjZp4DXuPuyTssiRDejGYUQQohUpCiEEEKkItOTEEKIVDSjEEIIkUrT9XWqwNy5c33BggWdFkMIIbqKu+++e7e7J1VNGGdKKIoFCxawbdu2ToshhBBdhZnlql4g05MQQohUpCiEEEKkIkUhhBAiFSkKIYQQqUhRCCGESKWnFcXo9lEWXLaAaZdMY8FlCxjdPtppkYQQonJMifDYRhjdPsrKW1ay/9B+AHbu3cnKW2rr5gwvGk77qBBC9BQ9O6MY2ToyriRC9h/az8jWkQ5JJIQQ1aRnFcWuvbsKbRdCiF6lZxXF0JyhQtuFEKJX6VlFseb0Nczsn7g888z+maw5fU2HJBJCiGrSs4pieNEw689az/w58zGM+XPms/6s9XJkCyFEHVNiPYrFixe7igIKIUQxzOxud1+cdVzPziiEEELkQ4pCCCFEKlIUQgghUpGiEEIIkYoUhRBCiFSkKIQQQqRSWUVhZu8ws4fM7BEz+1in5RFCiF6lkorCzPqAzwFnAK8Hft/MXt9ZqYQQojeppKIATgUecfdH3f0gcC1wdodlEkKInqSqiuIVwGOR/x8PtpWOFi8SQoh0qrpwkcVsm1BrxMxWAisBhoYaq/iqxYuEECKbqs4oHgeOj/x/HPBE9AB3X+/ui9198bx58xq6SNLiRau3rG7ofEIIMRWpqqL4F2Chmb3SzGYA7wX+seyLJC1SNHZgDLvEmHvpXJmihBA9TyUVhbs/D3wQuA34HnC9uz9Y9nWyFikaOzDGsk3LpDSEED1NJRUFgLtvdvcT3P3V7t6S1YSKLFIUKo2j//poKQwhRE9RWUXRDoYXDTM4MFjoM/sO7tMsQwjRU/S0ogC4/IzLsdggq2w0yxBC9AI9ryiGFw2zavGqps4RnWVopiGEmGr0vKIAWHvmWjYu3VjYDJWEnOBCiKmE1syOYXT7KKu3rGbswFhp54wyODDI5WdcrqQ+IURHybtmthRFBqPbRzn/lvN59tCzLTk/SHEIITpDXkUh01MGw4uG2fdn+0o1TdUjp7gQospIUeRkeNEwuz+6G7/Y2bh0I7P6Z5V+jdApfuGtF5Z+biGEaBQpigZo9Sxj3bZ1coQLISqDfBQl0ion+AWLL2DtmWtLPacQQshH0QHqzVNlzTbWbVsn/4UQomNoRtFGyphxaHYhhCgLzSgqSHTG0ahTfN22dXJ2CyHaihRFBwmd4hcsvqDQ567cdqXMUEKItiFFUQGKlhBxXKvwCSHahhRFRYiapfLMMMYOjMkEJYRoC1IUFSScYWT5L2SCEkK0AymKipLHfyETlBCiHUhRVJy1Z65N9V3IBCWEaDVSFF1A1ip8CpkVQrQSKYouIM8qfFIWQohWIUXRJWSZoEClPoQQraFyisLMPmVmPzKze4PXkk7LVBWyTFCgUuVCiPKpnKII+Ft3PyV4be60MFUhjwkqRKYoIURZVFVRiATWnrk2d8kPKQshRBlUVVF80MzuN7OrzOxFnRamahRRFkrKE0I0S0cUhZndaWYPxLzOBtYBrwZOAZ4E/kvCOVaa2TYz2/bTn/60jdJXg7zKwnFGto60QSIhxFSl0utRmNkC4CvuflLacd2yHkUrGN0+yvm3nM+zh55NPW7j0o0MLxpuk1RCiG6ga9ejMLOXRf49B3igU7J0A3lLla+8ZaVMUEKIhqicogAuNbPtZnY/8FvAhzstUDcQmqKSwmf3H9ovE5QQoiGmd1qAetx9eadl6FbWnrmW04ZOY9mmZbH7d+3d1WaJhBBTgSrOKEQTDC8aZv6c+bH7huYMtVkaIcRUQIpiCrLm9DXM7J85YdvM/pmsOX1NhyQSQnQzUhRTkOFFw6w/az3z58zHMObPmc/6s9Yr6kkI0RCVDo/NSy+HxwohRKN0bXisKI/R7aMsuGwB0y6ZxoLLFig8VgjREJWLehLlMLp9lJW3rGT/of0A7Ny7k5W3rASQCUoIUQjNKKYoI1tHxpVEiHIphBCNIEUxRUnKmVAuhRCiKFIUU5SknAnlUgghiiJFMUVRLoUQoiykKKYoyqUQQpSF8iiEEKJHUR6FEEKIUpCiEEIIkYoUhRBdiLLue5t2339lZgvRZSjrvrfpxP3XjEJUjjJGS1N5xN2tWfdT+Z60k07cfymKKU6nf5xFrx+Olnbu3Ynj46OlInKXcY5m2tDqc3Vj1n3Z96QKdOq31Yn7L0Uxhen0j7OR65cxWipzxBXXhvNuOo+5l86N7SDSOo+ylOA0i//ZtjvrvkhH2a2zoCSavZeNDKDC4ztx/zPzKMxsyN13ZW3rJMqjiGfBZQvYuXfnpO3z58xnx0U7KnP90e2jjGwdYdfeXTjxz6NhHLn4SK7rTrtkWux5ipwjJKkNUWb2z2T9WesBJtiOo/uGFw0nnqvP+thwzoZM+/KFt17IlduujG1b9DplEL0nQ3OGWHP6mvFzj24fZfWW1YwdGJv0ucGBQS4/4/JJcpR5T6pAM7+teh8DJN+/tO86SqP3v8w8iptybhMVo9MmijzXrx+ZJZF3tFT2iDvPdxWOjLNGzUnnOuyHM0ejo9tHE5VEn/WVriSSRsvhvqSOa+zA2IS2hCPhpHubdE86bTLNopnfVt7ZVdZ33Wd9bau6kBj1ZGYnAK8D5pjZOyO7jgFe0DKJRGkMzRmKHfW0y0SR5/pxP5p68taoCn9Yh/1ww+eIkzVrRgHpHUS4L+1cYUeR9GMf2TqS2Nke8SOldhJZHVnW/YoeWz9yjlJ/T8JZzM69OzFsvL1VjOpq5reVV8lk/TaO+JG2zcbSZhQnAv8ReCHw7sjr14Dzm7momb3bzB40syNmtrhu38fN7BEze8jM3t7MdXqdThcGzHP9tA626Ggp6YfVzIg7rg1xDM0ZyqzYm3WuPMom7fwwcSQ+99K5ib6UJEa3jyYqs117d+Weje7auyu1o6u/r9FZDDBJKVbNn9HMbytvZees77qdPqlEReHuN7r7cuBsd18eeV3o7l9v8roPAEuBr0U3mtnrgfdSU1LvANaaWV+T1+pZOl0YMM/1kx72+XPmc+TiI+y4aMeE49NMEkk/rGZG3PVtGBwYZEbfjAnHhB1EVucRnqsv4ZFO++En7TOMNaevYXT7KHMvncuyTcvGTUZjB8YYOzCW29kadtZJHDtwbO7OaWjOUOL9MGzSfc0zs8yrpJpxFOdVqM38tvIqmbTvut2VoPP4KM40s2PMbLqZ3WZmT5nZ+5q5qLt/z90fitl1NnCtuz/n7j8EHgFObeZavc7womF2XLQjttNtJeGPb/mm5QBcs/Sa2OsXGZllRZq0ag2O6He4+6O7uersq2I7iDydx/CiYTacs2FSm/un9bPv4L7EziruezKMVYtXAaTaskOyRuVZnfUzB59hycIlmTOsUHml3Y/6zjmPeW+aTSstvDq8vl1iLN+0vKHopUZ/W3mVTNIMdHBgsO2VoPNEPd3r7qeY2buA3wP+BLjT3U9u+uJmdwEfcfdtwf9XAN92943B//8D2OLuX047j6KeqkWRqI7w+KQImyhZkSZFr1v0+mUSveaxA8fyzMFnOHj4YKrcSXLm7WghPcooKTKpnsGBQYBUxeQXe+L9WHHyCjbct2HC9qhPIs/1zz3xXDY/vDn3dxGNRoqTK+74Naevaei5KPN5avWzmTfqKY+ieNDdTzSz9cBN7r45VB4Zn7sTeGnMrhF3vzk45i4mKorPAd+qUxSb3f2GmPOvBFYCDA0NvXnnznw/FNF6GgmLTfoR5AkPjHZ+RcM6Z/XP4tCRQ5mddJnUy7jv4L7Y9uUNY87bwaedc3T7KCtuXBEbCBDHzP6ZDEwfSJQ77GR37t1Jn/Vx2A9P2l5PEWURJ8/6s9azfNPyzDDcvIp1Zv/MhgYdRQcrnRiohJQZHrvZzB4AfgW4w8zmAs9lfcjd3+buJ8W8bk752OPA8ZH/jwOeSDj/endf7O6L582bl6MZvUu7Qw0bCYuNm/KPbh/lvJvOyzSphCaOpB9c1H5ff65nDz07QUlAaxynaaaOpPbltckXMa3t3LszMREwr5KAX0Q+xZkNlyxcMsExfdgPTzAnJnXSjo/7b+bPmZ9bllCeka0jucyPeb7XPuuLjfxavWV16m+paGJhp5Ni85Jr4SIzezGwx92fN7NZwAvd/UdNX3zyjOJE4O+p+SVeDmwFFrqnP8EyPSXTqDmmkeskjRRDoqPZPLOOuZfOzVQShnHN0muA+GS3ODNHHuJMNFkjv3pzEtTMM42OlPPOKPKYUuoJv5vND2/ObbaqJ/zu67+TpOdgcGCQA88fyBUOvf6s9ZnPU5I8Sc9BaKqaZtNSlWL9TCKPrOFzUDSxsGjiXv2sOCnBMS+lzSjMbDq1MNmNZnYt8AfAUw1J9YtznmNmjwNvAW41s9sA3P1B4HrgX4H/BXwgS0mIdNpROqE+tDGOvGGx4fbR7aOZSgJqo9DhRcOJ7Vx/9/rCSgImj9KzRn71+8OIo1DGohSJaql3juZh/6H9rNu2rmElAbXvKM6hm3Rvxw6M5boX+w/tZ9mmZew7uG9ShFmWPAAD0wfGtw0ODI4PFsJ7E6ckwu8tdCznndHU/5aKBlQUSdyLm2GPHRjj/Te/v+UzkDymp89Ry524Kni9BVjbzEWD0Nvj3P0od3+Ju789sm+Nu7/a3V/r7luauY5obXZ2aE5ZtmlZagdQn8eQJ3s6ryILf9BpWc9FieukkxTRsk3LWHDZAlZvWd2QQgoZHBhsKow57LCvWXpNbmWRh/lz5nPB4gsK5QyUFd8/dmAMd2dW/6zMY6Mmr2hHuufAHtZtW5d6bwYHBrlm6TX4xT6u7PLmz8DEZ69ofkXSd+X4JNPWyNYRDh05NOnYg4cPtjzHJI+i+FV3X+butwevP6DmrxBdQKtCRvPMIkIO+2FGto5MKAER14H3T+tn9/7d2CWW29kY/gCT2pOUs1BPqLgGBwYZmD7A8k3LJ/xQ0xRrmp8hDzP7Z3L5GZenhlrm9TOlZXAXlWnj0o3suGgHa89cWyhnICmUd/aM2YXlOHTkEHNnzmXj0o0Trn/B4gsmybP54c2TFEKe72LPgT3jCj/8XuPyZ/KUhimaX5GmkOpnrY0mZJZBnqin7wLnuPuO4P8FwI3u/saWSlYA+SiSaZWPokhIZvS6SZEyhmXajqOEETRAbNmH8HpFfBTTmMYRJtqRG7WXZxHKGpU5yd5c5B4WiYBKIvxum3k+4goY9k/rx8wmBA/Ebasnb+HAMtqeFJac5AMqI/w6y78X+ivSfnONFvosM+rpo8DXzOxOM9sKfBX4z4UlEh2hVdnZjYxg9h/anzjyTrIdRwlHuaGJAEgs+xC2M240HI5O66lXEqHMI1tHCpkishgcGGTV4lX0T+ufIPPYgTGWbVrG2774tgmzhzizVpKfKW2mmDW7is4i0kI542Y29duvf/D6SZ32oSOHOHrG0RPuxdXvuno8gTGJItngzRI1J4Zta6Y0TJ6optBsmGQy3Ll3J6PbR1lz+hr6p/VP2j+jb0bLs7TzRj0NUCsQaMC/uvuBlkpVEM0o2k8jM4pmiBvlJskQRr+k/YCLjD7DEW1ame84Zs+Yzb6D+yblERRNksuSK0rc6Ld/Wj/HHHUMew7siU3ug/TomaRifVB81pY2O2h29ttI9FcaRXIzkki6z4MDg+z+6O5cx0ZlAaoV9WRmv29mywDc/YC73+PudwPDZvaehqQSU4ZGR9iDA4Oxzr4w2zeOcFpd/2NImtU4zuotq1PlKDL6PHbgWEa3j7L+7vWFTBuDA4P4xc6GczYwf8788UJ5o9tHS7Epx7UhzrZuZuM1n0IH8eDA4IQZ1u6P7k5UElnF+opElsU5aZNkLzL7DZXZ/kP7E0fmM/pmxDrmkyiSm5EkU1LHP3ZgLFeZlnpZhhcNs/uju/GLHb/YE+9b2STOKMzsHuA33f3puu1zgP/t7m9uuXQ50Ywim1Zkf0bPmce/EB0V1csCcN5N502K6pjRN4Orzr6qUEmPkI1LN6aaUIqMPqdPm87zR57PdWyUwYHBSea2NF9NXvKOtJtdvKpVM8cyc3ny3stwFJ82Q6onLTcjK9s6T5mQ+nswun2UZZuWJX7GL24+UCFKGT6K6fVKAsDd9wKTDWWisrQq+zMaQ3/E06fgYabr6i2rWb1l9SSFNbxomKvfdfWEmcXgwGCikoDaCCwtFDQrZDAabz+rf1ZqzH6akkiy/RsWqwzCziPO3pxEnvDZOP9Bs+HRebOYi9LM0rT1bcxTeRZq0U3wi+fWL3auWXpNpn8k70wnKtuKG1dkyhRmyUfbMrxoOFEewzqWsZ02o/g34E3uvr9u+2zgbnd/bRvky4VmFOm0eknUtDpBWdm4zY4sL7z1QtZtWxe7L8mGnFasLulcSTRa5C4cqZ5/y/k8e+jZzOtk2aKT2pRWj6msGcUFiy8oLfs9jaQ25r1uWpvT/CMweQacJzIti/pnJKwGfNrQaYl+kbKXMS5jRnEV8A9mdlzkpMdRK7FxdfMiinbR6qS7tFXlIH1FtGazxNeeuTbRv5FkQ05Kntv88OZCNYbCqJfThk6blA2c5csIR6r7/mxfbI5AfZvqlxjN2yaIr8eUN0omyxc1ODA4IbKsCEWjlJKiv/LMaLLanDRrAHLNxvPOaqLUPyOOs27bOlZvWZ34/LRrGeN60hYuuhTYAvxzsAbFU8C3gdvd/dPtElA0T6uS7iA7dDCc7qfR7MN/+RmXF+oM0xRnljkrev4N52zgm7u+yfJNyyeM2p9+7unE5Kw42erLYKw9c21sclqaUk1q054De5oKjw470DhlHCYKRtuQZjap/2yRkM60ki6H/XCqGS8pibKe6H0Ia1bFVR2Iuw9lduBhfbA42rmqXZTUPAp3v8LdXwG8FviloOzGFe0RTZRFK5dEzVpVLs+DXcbCQkU6wzTFObxomFWLV036ofZP658QKRSONuPCZQ8dOZTos8m76EzRWWBWm5pZvCqMtInOfJI636RnbdXiVU3l8qTNOufPmc8xRx0Tu29W/yz2HNiTe6W/+pUCk6i/D2ERyHr6rG+8zWmRffWEyZhR2r2qXZRceRRVRz6KbFpV876RxYSipEU1NUta2fGsKJY831eRiKA+62PDORtyt7ORqqLtqBKcdq0VJ6/g+gevnzDybzbOPyQt72Xj0o2JNv0kkiKO8voZ+qyPI36EoTlDLFm4hC/c84XYOkzhtdacvqawjOFnW7lORV4fxfRSryoqSxhZVDZrTl8T22lE14kGEp3d0YJmZZc9j8oVjiSj10lTBHm+ryLmhqLrdsd9r4axZOGS2OPztKkskvwhcYEAzxx8ppRrDs0ZSkxcAwqVf4H4e1fEzxBea+fenZlJmOGzd+zAsYVCost2XDdD3jLjmdvE1Ca68M70v5iOXWIsuGwBwCSzz4qTVzCydWQ87A9IDZ9txWItWeXVy1hLvIjJLG6d6LT2Di8aZsXJKyaYHxxnw30bEtd/zlqfPI5GFrUqoiAPHj6YudhPHpJMWueeeG7hRZcg/t41GgqcZ5ZQNCS6k2amOPLUevpOzm1iilKfnRsdTYWj9KgTMFr7Pzxm1oz0UtGNRD+ldXKtjPQKSYoImj5t4jiqftW3vLksmx/eHJsJHf2emsmRafSzRX1KYwfGms7hSfJDxVWMhVqHnuQTMCy2E05rl2FsXLoxM18ojT0H9sTmCm1cunFS5FvePJl2kZZH8WLgZcC1wLkwPrQ5BviCu/9SWyTMgXwU+WjUT5Fli8+zcl0eisTVZ9nkW507EpUjLss876pvafLkWS2tmXbm+WxS++LMYs36CPISlSnpmkkZ1WGuwtoz105q25KFS1LzaPxiT60vltX+Rttcv6pdSBk+qDLyKM4ErqC2bvXnIq8/A/68YclER2hm5Jk1Ct+5d+f4KKfZFdOi8jazNnHeSK9mR2lxJqwiq76lfbfNrP+cZ+aUZ5XBuGcGJpsbw2q4eWl0ZlcvUxJJGdXXLL1mXEnUt23DfRsS18wIw37zRHUNDgxOyvJv1JQUypmU4d/qBYtCEn0N7n41cLWZnevu17dFGtEy0jrWrBFJkiMxSvhja3R96OgPKY8jOquTy+PczXOdNIrM0JK+wzRzR1agQKPnzfvZtGcmzgdy2tBpkyqbArGdXKMh0XkczvXBFPWRbMs3LY91fu8/tJ/BgUGO9B/JDM7Ik6ldRmBBVnvblYCXZ+GiDwJfdPenzexK4E3Ax919azsEzINMT9kUXfQ9StHyBHGlCeKuHQ0xjP6Q8phEyjAtJZ0jTyhr0XDURsNX8yx602hYbNZnm3lm8l4jr+kubEtWefike5f3GQ5NVu2IHstDVnubNaXmNT3lURT3u/sbzOx3gD8GLgbWq3psd9Fsx5q1ClfceaO23/paQGmdWZ4Oqoy8gbQfYda5Gvk+W5XL0sx50z5blp+nSD5L3Ip3eXxPIUlKLK9ZtEohqZAud3SdkUafpzIVxX3ufrKZ/S3wDXe/wcy+q6VQu4syE7IaGeUU6czydlDNdrxFnPT1lDHarjqtTuIr4tPKm8CZdM/yLFSVp3R43OynzIWE4q4Z195Z/bM4dORQokLNS5lLod5nZpuBs4AtQfXY7k/n7jHKXBI1zb6c5LTLyluIOpX3HdyXyxnYbC5EVsG7Zh3N3U6Zz0wcRezrUd9TWu2pJIdx0n2JltjIs75E1Pl93k3nseLGFRN8MGMHxnj/ze8vLXQ17h5sXLqRuTPnTlqlsJXO7Twzij7gzcAj7r7HzOYCx7v7dxu+qNm7gU9RW171VHffFmxfAHwPeCg49NvuvirrfJpRtJekUU6jo6ms5TtbaSfOKpE+e8bs2BlLO0tmtItWmcaSaGRGEaWIvM3er6IRfa02YZU1oy2thIe7HzazVwG/DawBBsg3E0njAWAp8PmYfT9w91OaPL9oIWWXi4iL7Dh05BCzZ8yetK5w2YQyxymqZw4+Mz5arI+IamfJjFYTF6dKNOf8AAAUnElEQVRfNAKsEeKiupJ8FEmz1CJVcKHx+1U0uqjV0UjNRLs1Qp4ZxRXUVrR7q7u/zsyOBW5z919u+uJmdwEfqZtRfMXdTypyHs0o8tPuUWMeqmDvr/9e9h3c19SCP91Cozb/Zq8Zftdh1dXozBGqp4CrNqMoa0ZbZlHAX3P3N5nZdwEC81PympHN88rgWk8Dn3D3r7fwWj1Fs3kDrZIpqaBbfQJeKzuP+tHptEviJ82dWjimbPJGsZXd3vpncOzA2HjC2uaHN7N80/LKKAeY+D3Vh3n3T+vniB+Z9OzO6JvR8jpN7Z7R5lEUh8xsGoED28wGgcxhnpndCbw0ZteIu9+c8LEngSF3HzOzNwM3mdmJcWt3m9lKYCXA0NDUcSC2kmaS7qD8zjprdbwiCXhl0+6pfTspkhdTdnuTnsFoBdYqDGBg8vcUrhHh+HjpcGht1FMaraoIHUceX8PngBuAeWZ2CfAN4DNZH3L3t7n7STGvJCWBuz/n7mPB+7uBHwAnJBy73t0Xu/viefPm5WiGaKbcQzMlQJLIWh0vOmrKs8pYmbRysadOk7ecdivam/SsxRU/LKPqbDPEfU9RJRFmec+eMZuNSzfiFzu7P7o71aHeyfY0Q6KiCEuJu/sXgU8AfwP8DHi3u1/bCmHMbF4QZUXgQF8IPNqKa/UizYR0tqKzzlodL+u4VpqBWh0aWgZFO57w+Dy29rwr8RWlyAwlT9XZVna+Sc9XKEuRQVMrBlrtJG1GMV5K3N0fdPfL3f0yd3+g2Yua2Tlm9jjwFuBWM7st2PVW4H4zuw/4MrDK3bMXXRa5aGaU3IrOOq/iKqrgyuo8ylizolUU7XjqS8UnEcbpp42MmyHuGcyzRjmUW2I9D2m5F0mDpqRnrxOz4jJJUxT57l4DuPuNwfrbR7n7S9z97cH2G9z9RHc/2d3f5O63tEqGXqTRUXLocI6jGRt2XsVVRMF1+8gtL0U7nixz08z+mWxcurHlCjHuGVy1eFVq4mOU6MCk1Z1v0nOXtEhS2kyjE7PiMklbj+Jx4L8mfdDdE/e1G4XHto4sx2ezzru8DvKskMqy6xNVnaIhxWklLEKbe6cdx0XDk9sRVh33fCZFi/VZX6wSCUuUV/G5LCM8tg+YTQtnFqL6ZI1Exw6MNRWhkjdyIzwuKwKq20dueSkalZV0fKc7qpD65yApT6DZEutFI/eSns842ZJ+J7v27opdRKm+PVXMcQpJMz096e5/4e6XxL3aJqHoKHk62HbaWrPMDb1QgwmK+5u6LYorj5m0aJvKMksmyRbOHOpJWkQp2p6qm0zTTE+VqhCbhkxPxSijkms97cqizjI3TMUaTEkUHYFWecTaKK2oStyMLI0+e50ymTZdZtzMju2WiCMpivyUseBOHO0yYTS6znO3d4iieTrl08jz7HWqjE3TZca7RUmIYhSNFKmfMpe5HnAj5DE3xIW1dnOyUz1TqS3tpB1myUZDqqtuMm22CqzoMhpx9kYf/t0f3c1VZ1/VsUS0RkJ8q27/LcJUaku7qbKfpsqyQY7qsd2ATE/56ZXw0ShTqc1TqS2doMpmyU7IVoaP4njgs8ArgC3AZ939ULDvJnd/V4nyNoUURX56ydkbUoUy5mUxldpSJaqsQFpJGUuhXgXcBXwIeBnw1aByLEB8HJioPN1Qw6hsqm7/LcJUakteWu2TkTkvmzRFMc/dr3T3e939Q8Ba4Gtm9mq0ZnZXU+UaRq2g6vbfIkyltuShHZ14t9dhagdpiqLfzF4Q/uPuG4HVwG3UZhhCdAXdMovKM3LulraURTs68V7J5m+GNB/Fh4F73P2rddvfCFzq7r/dBvlyIR+F6HZ60XeUh3b4ZKoWINBOf0kZeRR/W68kgu3fBW5tUj4hRASZP+Jph0+mSua8qvpLGs2j+JNSpRBtR0lb1ULmj3ja0YlXyZxX1QFDnjWz41BF2S6mE2tQi3TSKqH2augm/OJ5bHX727n+dBpVHTA0OqNQ1FMXU9VRSyvolplT0sh5ycIllTRFtJNeitKravhz2prZz5jZ0zGvZ4CXt1FGUTJVHbWUTVXtvXEkmT82P7y5Z5S6qJa/JIpKePQgVYvyaBVToZ3KxO49qhj11KiPQnQxa05fk7na1lRgKsycGlnFTXQ3VfGXRFH12B6kSlEeraSq9t4iVNUUIXoLzSh6lCqOWspmKsyc2hX1I0QaHfFRmNlngbOAg8APgPPc/efBvo8DfwQcBv7Y3W/LOp98FCKJXg4tFSKLpsuMtxIz+x3gf7v782b2GQB3/1Mzez3wJeBUapFVdwInuPvhtPNJUQghRHHKKDPeMtz9dnd/Pvj328BxwfuzgWvd/Tl3/yHwCDWlIYQQokNUwZn9fmoLI0FtkaTHIvseD7aJFtAtyWhCiM7SMme2md0JvDRm14i73xwcMwI8D4Q9VFxpkFjbmJmtBFYCDA11TxRLVVAZDyFEXlo2o3D3t7n7STGvUEmsAH4XGPZfOEoeB46PnOY44ImE869398XuvnjevHmtasaUpZfKeDSDZl1CdMj0ZGbvAP4UeKe7R3urfwTea2ZHmdkrgYXAdzoh41RnKiSjtZpuKgEiRCvplI/iCuBo4A4zu9fMrgRw9weB64F/Bf4X8IGsiCfRGFMhGa3VaNYlRI1ORT29xt2Pd/dTgteqyL417v5qd3+tu29JO49oHGX8ZqNZlxA1qhD1JDpAr5TxaAbNuoSooRIePUwvlPFohqlQAkSIMtCMQogENOsSoobWoxBCiB6l0iU8hBBCdA9SFEIIIVKRouhxlHkshMhCUU89jOo9CSHyoBlFD6PMYyFEHqQoehhlHnc/Mh2KdiBF0cMo87h6FOn4VbRQtAspih5G9Z6qRdGOX6ZD0S6kKHoYZR5Xi6Idv0yHol0o6qnHUb2n6lC04x+aM8TOvTtjtwtRJppRCFERivqMZDoU7UKKQoiKULTjl+lQtAsVBRSiQoxuH2Vk6wi79u5iaM4Qa05fo45ftIy8RQGlKIQQokdR9ViRGyVtCSHSkKLocZS01VtoUCAaQYqix1HSVu+gQYFoFCmKHkdJW72DBgWiUTqiKMzss2b2b2Z2v5ndaGYvDLYvMLMDZnZv8LqyE/L1Eqr31DtoUCAapVMzijuAk9z9DcD3gY9H9v3A3U8JXqs6I17voKSt3kGDAtEoHVEU7n67uz8f/Ptt4LhOyCGUtNVLaFAgGqUKtZ7eD1wX+f+VZvZd4GngE+7+9c6I1Tuo3lNvEN5jJfSJorQs4c7M7gReGrNrxN1vDo4ZARYDS93dzewoYLa7j5nZm4GbgBPd/emY868EVgIMDQ29eefOycXRhBBCJJM34a5lMwp3f1vafjNbAfwucLoH2srdnwOeC97fbWY/AE4AJqVdu/t6YD3UMrPLlV4IIURIp6Ke3gH8KfBOd98f2T7PzPqC968CFgKPdkJGIYQQNTrlo7gCOAq4w8wAvh1EOL0V+Aszex44DKxy9z0dklEIIQQdUhTu/pqE7TcAN7RZHCGEECkoM1sAqgEkhEimCuGxosOENYDC8g5hDSBAoZNCCM0ohGoACSHSkaIQqgEkhEhFikKoBpAQIhUpCpFaA0hObiGEFIVILAwIaKEbIUTraj21k8WLF/u2bZOqfIgmWXDZAnbunVxDa/6c+ey4aEf7BRJClEreWk+aUYhE5OQWQoAUhUhBTm4hBEhRiBS00I0QAqQoRIT6CCdAq98JIVTCQ9RIKuOx/qz1clwL0eNoRiEAlfEQQiQjRSEARTgJIZKRohCAIpyEEMlIUQhAEU5CiGSkKASQXMZDEU5CCEU9iXFCpTCydYRde3eNO7KlLITobTSjEOOEIbLRIoDLNi1j7qVzVQhQiB5GikKMExciCzB2YIzlm5Zz4a0XdkAqIUSnkaIQ46SFwjrOlduu1MxCiB6kY4rCzP7SzO43s3vN7HYze3mw3czsv5nZI8H+N3VKxl4jKxTWcSXgCdGDdHJG8Vl3f4O7nwJ8BfhksP0MYGHwWgms65B8PUeeUFgl4AnRe3RMUbj705F/ZwHhCkpnA1/0Gt8GXmhmL2u7gD3I8KJhBgcGU49RAp4QvUdHfRRmtsbMHgOG+cWM4hXAY5HDHg+2iTZw+RmXT0q8C1ECnhC9SUsVhZndaWYPxLzOBnD3EXc/HhgFPhh+LOZUk9ZrNbOVZrbNzLb99Kc/bV0jeoxo4h1An/UBKAFPiB6mEmtmm9l84FZ3P8nMPg/c5e5fCvY9BPymuz+Z9HmtmS2EEMWp/JrZZrYw8u87gX8L3v8j8AdB9NOvAnvTlIQQQojW0skSHp82s9cCR4CdwKpg+2ZgCfAIsB84rzPiCSGEgA4qCnf/vYTtDnygzeIIIYRIQJnZQgghUpGiEEIIkUolop6axcx+Ss3P0Shzgd0lidNJpko7QG2pKmpLNWm0LfPdfV7WQVNCUTSLmW3LEyJWdaZKO0BtqSpqSzVpdVtkehJCCJGKFIUQQohUpChqrO+0ACUxVdoBaktVUVuqSUvbIh+FEEKIVDSjEEIIkYoUhRBCiFR6WlGY2TvM7KFg2dWPdVqeJMxsh5ltD5aN3RZsO9bM7jCzh4O/Lwq2Jy4la2YrguMfNrMVbZL9KjP7iZk9ENlWmuxm9ubgu3kk+GxcmfpWtuVTZvaj4N7ca2ZLIvs+Hsj1kJm9PbI99rkzs1ea2T8HbbzOzGa0qB3Hm9k/mdn3zOxBM1sdbO+6+5LSlm68Ly8ws++Y2X1BWy5Ju76ZHRX8/0iwf0GjbczE3XvyBfQBPwBeBcwA7gNe32m5EmTdAcyt23Yp8LHg/ceAzwTvlwBbqK3r8avAPwfbjwUeDf6+KHj/ojbI/lbgTcADrZAd+A7wluAzW4Az2tyWTwEfiTn29cEzdRTwyuBZ60t77oDrgfcG768ELmhRO14GvCl4fzTw/UDerrsvKW3pxvtiwOzgfT/wz8H3HXt94ELgyuD9e4HrGm1j1quXZxSnAo+4+6PufhC4ltoyrN3C2cCG4P0G4F2R7XFLyb4duMPd97j7z4A7gHe0Wkh3/xqwpxWyB/uOcfdvee0X8sXIudrVliTOBq519+fc/YfUqiGfSsJzF4y4/x/gy8Hno99Lqbj7k+5+T/D+GeB71FaR7Lr7ktKWJKp8X9zd9wX/9gcvT7l+9H59GTg9kLdQG/PI1suKopuWXHXgdjO728xWBtte4sE6HcHfFwfbk9pVpfaWJfsrgvf129vNBwOTzFWhuYbibRkEfu7uz9dtbymBueKN1EavXX1f6toCXXhfzKzPzO4FfkJN8f4g5frjMgf79wbylt4H9LKiyLXkakU4zd3fBJwBfMDM3ppybFK7uqG9RWWvQpvWAa8GTgGeBP5LsL3ybTGz2cANwEXu/nTaoTHbqt6Wrrwv7n7Y3U8BjqM2A3hdyvXb1pZeVhSPA8dH/j8OeKJDsqTi7k8Ef38C3EjtAXoqmOIT/P1JcHhSu6rU3rJkfzx4X7+9bbj7U8GP+wjw36ndGyjelt3UTDrT67a3BDPrp9axjrr7pmBzV96XuLZ0630JcfefA3dR81EkXX9c5mD/HGqm0fL7gFY4ZbrhRW3RpkepOXtCx86JnZYrRs5ZwNGR9/+Hmm/hs0x0PF4avD+TiY7H7wTbjwV+SM3p+KLg/bFtasMCJjqAS5Md+Jfg2NBpuqTNbXlZ5P2HqdmGAU5kokPxUWrOxMTnDvgHJjotL2xRG4ya3+Cyuu1dd19S2tKN92Ue8MLg/QDwdeB3k65PbYG3qDP7+kbbmClbK39UVX9Ri+b4PjU74Ein5UmQ8VXBDb0PeDCUk5otcivwcPA3/IEa8LmgTduBxZFzvZ+aY+sR4Lw2yf8lalP/Q9RGNH9UpuzAYuCB4DNXEFQbaGNbrglkvZ/aeu/RDmokkOshIlE/Sc9dcK+/E7TxH4CjWtSOX6dmcrgfuDd4LenG+5LSlm68L28AvhvI/ADwybTrAy8I/n8k2P+qRtuY9VIJDyGEEKn0so9CCCFEDqQohBBCpCJFIYQQIhUpCiGEEKlIUQghhEhFikJMGcxsMFIt9Md11UNzVfw0s6vN7LUZx3zAzIZLkvkbQTXPUM7ryjhv5PyPm9kLyzyn6D0UHiumJGb2KWCfu/9N3Xaj9twf6YhgdZjZN4APuvu9LTr/48BJXsv0FaIhNKMQUx4ze42ZPWBmVwL3AC8zs/Vmti2o+//JyLHfMLNTzGy6mf3czD4drA/wLTN7cXDMX5nZRZHjPx2sI/CQmf1asH2Wmd0QfPZLwbVOKSDzRjNbZ2ZfN7Pvm9kZwfYBM9tgtbUe7gnrfgXy/m3QzvvN7MLI6S4ys+8G209o+gsVPYcUhegVXg/8D3d/o7v/iFqpisXAycBvm9nrYz4zB/iqu58MfItaFnIc5u6nAv8ZCJXOh4AfB5/9NLWqpklcFzE9fTqy/XjgN4CzgPVmdhTwx8BBd18ELAeuCcxqFwAvB0529zdQKyEd8pS7vxH4AvAnKXIIEcv07EOEmBL8wN3/JfL/75vZH1H7DbycmiL517rPHHD3LcH7u4H/kHDuTZFjFgTvfx34DIC732dmD6bI9p4E09P1gYnsITN7DFgYnPezwXkfNLMngNcAb6NW7+hwsC+6bkZUviUIURApCtErPBu+MbOFwGrgVHf/uZltpFY3p56DkfeHSf69PBdzTBlLf9Y7EJNKRYfXS3I4xsknRG5kehK9yDHAM8DTkZXayuYbwLkAZraI2oylKO+2GidQM0M9DHwNGA7O+zpqS4E+AtwOXGBmfcG+Y5tugRABGl2IXuQeamamB6iVXf5mC67xd8AXzez+4HoPUFuBLI7rzOxA8P4pdw8V1yPUFMOLgZXuftDM/g74vJltp1bF9g+C7Z+nZpq638yep7Zwz5UtaJfoQRQeK0QLCBaSme7u/x6Yum4HFvovlrTM+vxG4MvuflMr5RQiD5pRCNEaZgNbA4VhwPl5lYQQVUMzCiGEEKnImS2EECIVKQohhBCpSFEIIYRIRYpCCCFEKlIUQgghUvm/i1/pGfg0AxwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd8079a3a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "\n",
    "# start the session\n",
    "with tf.Session() as sess:\n",
    "    #    initialize the variables\n",
    "    sess.run(init_op)\n",
    "    mc_losses = []\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        \n",
    "        current_start = epoch*batch_size\n",
    "        data_train_batch = data_train[current_start:(current_start+batch_size),:,:]\n",
    "        start_train_batch = start_train[current_start:(current_start+batch_size),:]\n",
    "        \n",
    "        # begin in state of all zeros\n",
    "#         begin_state = np.zeros((batch_size, state_size))\n",
    "        # begin in randome state\n",
    "#         begin_state = np.random.rand(batch_size,state_size)\n",
    "        begin_state = start_train_batch\n",
    "\n",
    "        _,cost,est_data_batch = sess.run([optimizer, estimation_cost, est_data_series], \n",
    "                                          feed_dict={batch_data_placeholder: data_train_batch, \n",
    "                                                     init_state: begin_state,\n",
    "                                                    batch_start_placeholder: start_train_batch})    \n",
    "\n",
    "        if epoch % 100 == 0: \n",
    "            \n",
    "#             begin_state = np.random.rand(batch_size,state_size)\n",
    "            begin_state = start_test\n",
    "            mc_cost, mc_estimation, squared_error = sess.run([estimation_cost, est_data_series, \n",
    "                                                              squared_difference], \n",
    "                                             feed_dict={batch_data_placeholder: data_test, \n",
    "                                                        init_state: begin_state,\n",
    "                                                       batch_start_placeholder: start_test})\n",
    "            \n",
    "            \n",
    "#             plt.plot(epoch, mc_cost, 'bo')\n",
    "#             plt.plot(epoch, zero_force_error, 'ro')\n",
    "#             plt.plot(epoch, mse_error, 'go')\n",
    "            print('Epoch {}, Train Cost {}, Test Cost: {}'.format(epoch, cost, mc_cost))\n",
    "            plt.plot(epoch,np.log(cost),'go')\n",
    "#             print('Epoch {}, Train Cost {}'.format(epoch, cost))\n",
    "    \n",
    "    \n",
    "    plt.xlabel('Training Epoch')\n",
    "    plt.ylabel('L2 Test Cost')\n",
    "    plt.title('NN Equalizer: no noise')\n",
    "    # plt.text(1000, .025, r'NN equalizer')\n",
    "    # plt.text(0.5, .025, r'Zero Force equalizer')\n",
    "    plt.show()\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimation error for test point k= 93 : 1.663869095242362e-08\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4lGX2//H3SSeEntCJoYQO0sGg9N5RQLCBLMKKKKtfXV3RtS1rWdRdG0WICrIUUREBRarAUkPvvQVCSYCQkEJC7t8fM/iLmJA2M88kc17XNVdmnnnKZyI5Pm3uI8YYlFIqP7ysDqCUKry0gCil8k0LiFIq37SAKKXyTQuIUirftIAopfJNC4hSKt+0gCil8k0LiFIq33ysDpAfwcHBJiwszOoYShVZ27ZtizXGhOQ0X6EsIGFhYURFRVkdQ6kiS0RO5WY+PYRRSuWbFhClVL5pAVFK5ZsWEKVUvmkBUUrlm0MKiIhEishFEdmbzfsiIh+JyFER2S0izTK9N1xEjtgfwx2RRynlGo7aA/kS6HGH93sC4fbHaGAygIiUBV4DWgOtgNdEpIyDMimlnMwh94EYY9aKSNgdZukPzDS28RM3iUhpEakEdACWG2MuA4jIcmyFaI4jcinXyTAZJKUlkZCaQOKNxN8eN81NfLx8/vAo5lOM4MBgSvqXRESsjq/yyVU3klUBzmR6HW2flt30PxCR0dj2XggNDXVOSpUlYwxnE86y/9J+zsSf4cy1M0Rfi/7tZ/S1aBJvJFLMpxhBfkGU8C9BkF8QxX2L4+PlQ3pG+m+PtIw00jPSSUpLIi4pjpT0FIIDgwkODCakeAjli5enRukahJcLJ7xsOOHlwgkJDNEi46ZcVUCy+q9v7jD9jxONmQZMA2jRooWOBO0kyWnJ7Di/gz0X9rD7wm72XNzD3ot78fHyoUH5BoSVDqNqiaq0qtKKB+o9QNWSValasiqlAkrhJXk/Ik5JTyEuKY7YpFhik2I5n3ieY1eOsfLESqZETeHI5SOkZ6QTXjacphWb0rJKS1pVaUWDkAb4evs64Teg8sJVBSQaqJbpdVXgnH16h9umr3FRJgUkpCaw4cwG1p5ay9rTa9kRs4O6wXVpXKExjco3YmC9gTQq34gKQRWcsv0AnwCqlKxClZJZ7ngCcDn5ModiD7E9Zjsbzmzg35v+zen409xd8W5aVm5J+7va07lGZ0r6l3RKRpU9cVRbB/s5kMXGmIZZvNcbGAf0wnbC9CNjTCv7SdRtwK2rMtuB5rfOiWSnRYsWRr8Lkz/GGLbFbGPhwYUsO7aMA5cO0Lxyc9qFtqN9WHvaVG1DkF+Q1TFzdC31GtvObWPz2c2sOrGKjdEbaVKxCd1rdqd7ze40q9QMby9vq2MWWiKyzRjTIsf5HFFARGQOtj2JYOACtisrvgDGmCliO4D9BNsJ0iTgcWNMlH3ZkcDL9lVNNMZ8kdP2tIDkzY2bN/j15K8sPLiQRYcXEegbyIA6A+gV3ovWVVsT4BNgdcQCS0pLYu2ptSw7uoxlx5Zx8fpF+tbpy7CGw+hUvRM+XoXye6OWcWkBcTUtIDkzxrDhzAYid0Ty/cHvqRNch/51+jOg7gDqBte1Op7TnY4/zbf7v2XuvrmcvHqSQfUGMazRMCKqReTrXI2n0QLioWISYpi5ayaROyMRhJFNR/JI40eoXKKy1dEsc+zyMebuncvcfXOJT4nn8SaPM7r56Dued/F0WkA8iDGG5ceX8/GWj1l/ej2D6g1iZNORtKnaRi9/3mb3hd1MjZrKnL1z6FyjM0+1fIr2d7XX39NttIB4gLSbaczdO5dJGyeRYTJ4ts2zPNjgQYr7Fbc6mtu7lnqNWbtm8enWT/ESL55q+RQjmoygmG8xq6O5BS0gRdi11Gt8vu1z/r3534SXDeeFiBfoUauH/l80H4wxrD65mv9s/g9bz27lhYgXGNNiDIG+gVZHs1RuC4ieTSpEktOSeXf9u9T8qCZbz21l4YMLWTV8FT3De2rxyCcRoVP1Tvww9AeWPryUdafXUfOjmry/4X2u37hudTy3pwWkEEjPSOfzbZ8T/nE4W89tZd3j65g7aC7NKze3OlqR0qRiE7578DuWPbKMjdEbqflRTT7Y+AE3bt6wOprb0gLixowxfHfgOxp+1pD/7v0v3w75lgVDFnjEZVgrNa7QmAVDFrD80eWsOrGKhp81ZPHhxRTGw31n03MgburApQM8ueRJrqZc5Z0u79C9Znc9TLHIT0d+4tllzxJWOowPu39IvZB6VkdyOj0HUkglpyXzyqpXaPdlOwbVH8S20dv0BKnFeob3ZM+Te+hRqwftvmzH+J/Gcy31mtWx3IIWEDey7OgyGk5uyOG4w+z68y7GtRqn3+dwE77evvylzV/YP3Y/iTcSaTS5EcuPLbc6luX0CwJu4EryFcb9NI6NZzbyaa9P6Rne0+pIKhshxUOY0X8Gy44uY9SPo+heszuTuk3y2G8C6x6IxX49+StNpjahXLFy7B27V4tHIdG9Vnd2/3k3AI0mN+KXY79YnMgaugdikbSbaby+5nUid0Yyo98MeoX3sjqSyqNSAaWY1ncavxz7hSd+fIL+dfozqdsk/Lz9rI7mMroHYoEjcUdoG9mWnRd2snPMTi0ehVy3mt3YOWYnp+NPc98X93Hqaq7ayhYJWkBc7IeDP9A2si2P3f0Yi4ctdtpIX8q1yhQrw/cPfs+Q+kNoNb0VSw4vsTqSSzjkEEZEegD/AbyB6caYd257/0Ogo/1lIFDeGFPa/t5NYI/9vdPGmH6OyORuMkwGE9dOZNr2aSx+aDGtqrSyOpJyMBHh/yL+jzZV2zD026E8cvoR3ur0VpEezKjAN5KJiDdwGOiKbYzTrcAwY8z+bOZ/GmhqjBlpf51ojMnTGHqF7UayxBuJjFg4grMJZ/luyHdUKlHJ6kjKyS5dv8RD3z2Ej5cP8wfNp4R/Casj5YkrbyRrBRw1xhw3xtwA5mLrA5OdYXhQ35cTV04QMSOCkv4lWTN8jRYPDxFSPISfHv6J0JKh3PfFfURfi7Y6klM4ooDkpbfLXUB1YFWmyQEiEiUim0RkgAPyuI3tMduJiIxgVLNRzOg3A38ff6sjKRfy8fJhSp8pPNToISJmRLDr/C6rIzmcIw7Oct3bBRgKLDDG3Mw0LdQYc05EagCrRGSPMebYHzZSyBpL/XryVwZ/M5gpfaZwf737rY6jLCIi/LXtXwkrHUbXWV2ZOXAmPWrdqQts4eKIPZDser5kZSi3Hb4YY87Zfx7H1hOmaVYLGmOmGWNaGGNahISEFDSzUy06tIhB3wxizgNztHgoAIY0GMLCoQsZsXAE8/bOszqOwziigGwFwkWkuoj4YSsSi26fSUTqAGWAjZmmlRERf/vzYKAtkOXJ18Ji5q6ZjP5xNEsfWkrnGp2tjqPcSES1CJY/upxnlz3L7N2zrY7jEAU+hDHGpIvIOGAZtsu4kcaYfSLyJhBljLlVTIYBc83vL/vUA6aKSAa2YvZOdldvCoOpUVOZuG4iq4ev9oivfKu8a1ShESseW0HXWV1Jz0hneJPhVkcqEB0PxEFm7prJhFUTWDN8DTXL1rQ6jnJzB2MP0mVmF97s+CYjm460Os4f5PYybtG9w8XJZs+Gk+Mmce5qIPPbVSCt64tsHLNKi4fKlbrBdVk1fBWdZ9oOc92xiOSGFpB86NIFKq78mq95gaXh8FWrQG5OX8/26vWo97DV6VRhUbtcbVY+tpL2X7YnJDCEvnX6Wh0pz/S7MHnUpQuwcgVf8DhrwmDEAFgxN4l3T33BhAlWp1OFTe1ytflh6A+MXDSSTdGbrI6TZ1pA8mDsWIhZuY9veYCjwekMGQzzvoGm0X58z0BOn7Y6oSqMWlVpxVcDvmLA3AEcij1kdZw80QKSS2PHwjeTL7GYPqQXu0bfh+C95dDxJDzGTNbQkUJwf5tyU73Ce/FOl3foMbsHMQkxVsfJNS0guTB7NkyffINveYAq3ie5/0F4YD+M2AnP8y/m8yAAEydaHFQVaiOajGBU01H0mdOH5LRkq+PkihaQXBgzBj7gOe5jHU/2hjIp8PZKmMIY3uf/AOjcGR7WE6iqgF6+72XCy4YzdunYQtGHRgtIDsaOhQHXv2Ycn/LvNrCtMnz9HawxHXmajwGhc2dYscLqpKooEBGm95vO1rNbmbptqtVxcqSXce9g7FhYM3k/WxnDxqrwzr2w5XO4dCOMIcwnHV+CgrR4KMcK8gvi+we/p21kW5pUbEKbqm2sjpQt3QPJxuzZ8OXkJOYzhBsBSQwbBJ//COXjA7if74gjGIApUywOqoqk8HLhzOg3g8HfDOZC4gWr42RLC0g2xoyBSTxPA/Yxqh/0Pwj9DsE4PmGn/QvDTz6p5z2U8/St05fhdw/n0e8fJcNkWB0nS1pAsjB7NrS7vpSxTGZySzhRxnbJdhaPEIntluMnn4TPPrM4qCryXu/wOvGp8UzeOtnqKFnSL9PdZvZsGP9IHHtoyKUK5+n8GGyYAXK5Jk3ZQSIlCAqChASnbF6pPzgcd5i2kW1Z//h66gTXcck2tbl2PsyeDcOHw394hhCv8zzeH95dAdUve/MQ/yUR28C4et5DuVLtcrV5vf3rPLbwMdIz0q2O8ztaQDIZMwZ63FzMw/yXD9tA2WR4fAdMZAJbsbVh0PMeygpPtnySUv6leHvd21ZH+R0tIHazZ4NcT2AyT3K0LLx7L0xdDLu4m3/wCgBBQXreQ1nDS7yI7B/Jx1s+ZveF3VbH+Y1DCoiI9BCRQyJyVEReyuL9ESJySUR22h+jMr03XESO2B+WDc80fjy8wWtUJZrRfeFv6yD0ijcjiSQdX0APXZS1qpasypsd3+SppU+5zV2qBS4g9sZSnwI9gfrAMBGpn8Ws84wxTeyP6fZlywKvAa2x9Zd5TUTKFDRTflSJ28UzfERkU0jwg/Gbbbev76AZoIcuyj080ewJktKS+Hr311ZHAaxpLJVZd2C5MeayMeYKsBywZMz7p/mYhICb/K2L7Yax6Iy7eIPXAL1kq9yHt5c3n/X6jBdXvEh8SrzVcVzaWOoBEdktIgtE5FYbiFw3pXK2CWWn0KVdB3of9KHJeXiGj0iiuJ73UG6nddXW9ArvxetrXrc6ikMKSG4aS/0IhBljGgMrgK/ysKxtRpHR9g52UZcuXcp32Oy89P5ptjfZzU+rt/A8/+JH+uHnp+c9lHt6u/PbzN4zmz0X9uQ8sxO5pLGUMSbOGJNqf/k50Dy3y2Zah1MbS20OepkHqv6FgHJN+UCe5667IDJSz3so9xRSPIQJ901gwiprx9F0SWMpEcncUbofcMD+fBnQzd5gqgzQzT7NpTZFb2L96fV8NeY5Tp6EjAw4eVKLh3JvY1qMYcf5HWyO3mxZhgIXEGNMOnCrsdQBYP6txlIi0s8+2zMisk9EdgHPACPsy14G3sJWhLYCb9qnuYwxhheWv8A/Ov2D4n7FXblppQokwCeAV9u9yqurX7Usg8d/F2bNyTWM/nE0B546gLeXt0PWqZSrpN1Mo+6ndYnsF0n7sPYOW69+FyaX/rnun7x070taPFSh5Ovty2vtX+PV1a9acnOZRxeQrWe3cjD2II80fsTqKErl28ONHubi9YusPrna5dv26ALy9vq3eT7iefy8/ayOolS+eXt583zE83y46UOXb9tjC8j+S/v535n/MarZqJxnVsrNPdzoYTZHb+ZI3BGXbtdjC8gHGz/g6VZPE+gbaHUUpQqsmG8xRjcfzUebP3Lpdj2ygMSnxLNg/wKeaPaE1VGUcpixLccye89srqZcddk2PbKAzN4zm641u1IhqILVUZRymMolKtMzvCczts9w2TY9roAYY5i6bSpjmo+xOopSDvdMq2eYHDXZZZd0Pa6AbDm7hcQbiXSq3snqKEo5XKsqrfD19mVj9EaXbM/jCsi0bdMY3Ww0XuJxH115ABHh0caPMmvXLJdsz6P+ipLTkvn2wLcMb2LZyIlKOd3DjR5m/v75pKan5jxzAXlUAVl2bBnNKzenYlBFq6Mo5TR3lb6LxhUas+TIEqdvy6MKyIL9CxhUb5DVMZRyukcbP8qs3c4/jPGYApKansqSI0sYWG+g1VGUcroH6j3AiuMrSLyR6NTteEwBWX58OY0rNNbDF+URSgWUonWV1qw4vsKp2/GYAqKHL8rT9KndhyWHnXsexFWNpZ4Tkf32UdlXishdmd67manh1KLbl3WEDJPBkiNLGFB3gDNWr5Rb6h3em6VHlzr1pjJXNZbaAbSwj8q+AHgv03vJmRpO9cMJdl/YTdliZalWqlrOMytVRISXC6e4b3F2nt/ptG24pLGUMWa1MSbJ/nITttHXXWbViVV0rt7ZlZtUyi30Du/N4sOLnbZ+VzaWuuVPwE+ZXgfY+71sEpFsjzEK0hdm5YmVeuu68kg9avVg+fHlTlu/qxpL2WYUeQRoAfwr0+RQ++CtDwH/FpGaWS2b374waTfTWH96PR3COuR6GaWKijZV27A9ZjtpN9Ocsn6XNJYCEJEuwASgX6YmUxhjztl/HgfWAE0dkOk3W89tpUaZGgQHBjtytUoVCqUCSlGjTA2nnQdxVWOppsBUbMXjYqbpZUTE3/48GGgL7HdApt+sO7WOdqHtHLlKpQqViGoRTvt2rqsaS/0LCAK+ue1ybT0gyt5wajXwjjHGoQVkx/kdtKicY3sLpYqsiGoRbDizwSnr9nHESowxS4Glt037e6bnXbJZbgPQyBEZsrM9ZjuvtHvFmZtQyq3dU/UeXlnlnL+BIn0nakJqAmcTzlI3uK7VUZSyTK2ytbiedp3ziecdvu4iXUB2XdhFw/IN8fFyyI6WUoWSiFA/pD4HLh3IeeY8KtIFZEfMDppWdOhFHaUKpXrB9TgQqwUkT3Zd2MXdFe62OoZSlqsXXE/3QPLq+JXj1Cpby+oYSlmuXojugeTZyasnCSsdZnUMpSynhzB5lJ6RztmEs4SWCrU6ilKWq1aqGldTrpKQmuDQ9RbZAhJ9LZryxcvj7+NvdRSlLOclXlQuUZmYxBjHrteha3MjJ66coHrp6lbHUMptVAyqSEyCFpBcORV/Ss9/KJVJxaCKDr+ZrMgWkNikWEICc/+1f6WKukpBlfQQJreuplyldEBpq2Mo5TZ0DyQPtIAo9Xu6B5IHV1OuUqZYGatjKOU2SvqXdHijqSJdQHQPRKn/L8AngOS0ZIeu01V9YfxFZJ79/c0iEpbpvb/Zpx8Ske6OyANwJeUKpfxLOWp1ShV6AT4BpKSnOHSdruoL8yfgijGmFvAh8K592frYhkBsAPQAPrOvr8Bu3LxBgE+AI1alVJHglgWEXPSFsb/+yv58AdBZRMQ+fa4xJtUYcwI4al+fUsrB3LWA5KYvzG/z2MdQjQfK5XJZpZQDpN5MZcf5HQ5dp6v6wmQ3T156yuS7sZRSyvYFU0dzVV+Y3+YRER+gFHA5l8sC+W8spZSyKeFXgiYVmzh0nS7pC2N/Pdz+fBCwythahi8Chtqv0lQHwoEtDsiklLpNSnqKwy8sFHi0YWNMuojc6gvjDUTe6gsDRBljFgEzgFkichTbnsdQ+7L7RGQ+tmZS6cBTxpibBc0E4OPlQ1qGc9r5KVUYJacnu18BgVz1hUkBBmez7ERgoiNyZFY6oDRXU646erVKFVrO2AMpsneiagFR6ve0gORBaX8tIEpllngjkeK+xR26ziJbQMoUK6MFRKlMzieep2JQRYeus8gWkNIBpbmSfMXqGEq5DS0geVC2WFnikuOsjqGU24hJjKFSUCWHrrPIFpDQUqGcij9ldQyl3IbugeRB9dLVOXHlhNUxlHIbMQkxVCqheyC5EloqlLMJZ51y/79ShY0xhnMJ5/QQJrf8ffwJCQzh7LWzVkdRynLnE8/j7+Pv8GE+i2wBAahepjonruphjFIHYw9SL7iew9dbpAtIjTI1OH7luNUxlLLcgdgD1A2u6/D1FukC0jCkIbsv7LY6hlKWO3DpgO6B5FXTSk0dPgKTUoXRgdgD1AvRApInTSs2Zef5nWSYDKujKGWp/Zf26x5IXpULLEfpgNJ6HkR5tOhr0aRlpBFaKtTh6y7SBQRseyE7YvQwRnmujWc2ElEtAlsjBMcqUAERkbIislxEjth//uEis4g0EZGNIrJPRHaLyIOZ3vtSRE6IyE77w7EDNmIrINtitjl6tUoVGhvObOCeqvc4Zd0F3QN5CVhpjAkHVtpf3y4JeMwYc6t51L9FJHPPyReMMU3sj50FzPMHbUPbsu70OkevVqlCY0P0BiKqRThl3QUtIJkbRn0FDLh9BmPMYWPMEfvzc8BFwGXDqkdUi2DX+V0kpCa4apNKuY3ktGT2XtxLi8otnLL+ghaQCsaYGAD7z/J3mllEWgF+wLFMkyfaD20+FBH/Aub5g0DfQFpWacn60+sdvWql3F7UuSgahDQg0DfQKevPsYCIyAoR2ZvF4/b2lTmtpxIwC3jcmN+uq/4NqAu0BMoCL95h+Xw3luoU1omVJ1bmaRmlioJfjv1Cx7COTlt/jgXEGNPFGNMwi8cPwAV7YbhVIC5mtQ4RKQksAV4xxmzKtO4YY5MKfMEd+uIWpLFU5xqdWXViVZ6WUaooWHJkCX1q93Ha+gt6CJO5YdRw4IfbZ7A3m/oemGmM+ea2924VH8F2/mRvAfNkqWXllhy7cozYpFhnrF4pt3Qu4Ryn4k9xTzXnXIGBgheQd4CuInIE6Gp/jYi0EJHp9nmGAO2AEVlcrp0tInuAPUAw8I8C5smSr7cvXWp0YcnhJc5YvVJuaemRpXSr2Q0fL4e0f8pSgdZsjIkDOmcxPQoYZX/+NfB1Nst3Ksj282JQvUHM3jOb4U2G5zyzUkXA4sOLGVR/kFO3UeTvRL2ld+3erDu9jviUeKujKOV0KekprD65mh61ejh1Ox5TQEr6l6RDWAd+PPyj1VGUcrolh5fQvFJzggODnbodjykgYDuMWbB/gdUxlHK6Wbtn8WjjR52+HY8qIH3r9GXViVVcS71mdRSlnCY2KZbVJ1fzQP0HnL4tjyogpQNK07VmV+bsmWN1FKWcZt7eefQO701J/5JO35ZHFRCA0c1GM237NKtjKOU0rjp8AQ8sIF1rduVy8mWizkVZHUUphzsUe4iTV0/StWZXl2zP4wqIl3jxRLMnmLZN90JU0fPJlk8Y2XSkU28ey8zjCgjAyKYj+Wb/N3oyVRUpV1OuMnvPbJ5q+ZTLtumRBaRiUEU6Ve/E17uzvEFWqUIpckckPcN7UqVkFZdt0yMLCMBzbZ5j0oZJ2jtXFQnpGel8tPkjxrce79LtemwBaRvalmqlqjFv7zyroyhVYIsOLaJKySq0qpLtiBhO4bEFBODle1/m7fVva98YVagZY5i0YRJ/af0Xl2/bowtIt5rd8PfxZ/HhxVZHUSrffj76M/Gp8dxf736Xb9ujC4iI8PK9LzNx3USMMVbHUSrPjDG8svoV3uzwJt5e3i7fvkcXEICB9QaSkJrAT0d/sjqKUnm28OBCjDEMrDfQku07vbGUfb6bmUYjW5RpenUR2Wxffp59+EOX8hIv3u3yLi8sf0GvyKhC5WbGTV5d/SpvdXwLL7FmX8AVjaUAkjM1j+qXafq7wIf25a8AfypgnnzpU7sP5YuXJ3JHpBWbVypf5u+bTwn/EvQK72VZBqc3lsqOfSDlTsCtATrytLwjiQiTuk7itTWvaQMqVSgkpSXxt5V/4+3Obzul521uuaqxVIC9p8smEblVJMoBV40xt44bogHX3UJ3m+aVm9O1Rlfe+997VkVQKtfeXvc2baq2oUNYB0tz5PiNGxFZAVTM4q0JedhOqDHmnIjUAFbZR2LP6oso2V4KEZHRwGiA0NDQPGw69/7R6R80ndqUJ5o/QWgp52xDqYI6EneEyVGT2fXnXVZHcU1jKXtPXIwxx4E1QFMgFigtIreKWFXg3B1y5LuxVG6FlgrlL63/wpNLntTLusotGWMY//N4Xmz7oku/85IdVzSWKnOr562IBANtgf3G9he6Ghh0p+Vd7cV7X+R0/Gnm7p1rdRSl/uCHQz9w4uoJxrdx7XdesuOKxlL1gCgR2YWtYLxjjNlvf+9F4DkROYrtnMiMAuYpMD9vP6b3nc6zy57VTnbKrcSnxDP+5/F80vMT/LxdfsdDlqQw7qq3aNHCREU5d0SxZ39+lrjkOGYOnOnU7SiVW4//8Dh+Xn5M7TvV6dsSkW3GmBY5zefxd6Jm561Ob7Hu9Dp+Pvqz1VGUYuHBhaw9tZb3u79vdZTf0QKSjSC/ID7v+zmjFo3i4vUszw0r5RIXr1/kySVPMnPATIL8gqyO8ztaQO6gS40uPNL4EUYsHKFf+VeWMMYw+sfRDL97OG1D21od5w+0gOTgrY5vcSXlCh9u/NDqKMoDTd8+nRNXT/BGhzesjpIlLSA58PX2Zc4Dc3j3f++y5ewWq+MoDxJ1LooJqyYwb9A8/H38rY6TJS0guRBWOowpfaYwdMFQ4lPirY6jPEBsUiyD5g9iSp8p1A2ua3WcbGkByaX7691Pn9p9GPbtMP3av3Kq9Ix0hi4YytCGQy0ZZSwvtIDkwfvd3ictI43nf3ne6iiqCHtl1SuA7btZ7k4LSB74evvyzeBv+Pnoz0yNcv7NPMrzzNkzhzl75zDngTku6y5XEO6f0M2UDijN4ocW0zayLbXK1qJzjc5WR1JFxMrjKxn/83hWPraSkOLO+cKoo+keSD7UKluLeYPm8dB3D3Ew9qDVcVQRsPP8ToZ9O4z5g+fTqEIjq+PkmhaQfOoQ1oH3urxHt1ndOHHlhNVxVCF28upJ+vy3D5/2+tTyAYLySg9hCmB4k+Ek3kiky6wurB2x1i3GZ1CFS1xSHD2+7sFf2/6VwQ0GWx0nz7SAFNBTrZ7ietp1uszqwq8jfqV88exGdVTq9+KS4ugyqwsD6w7kmdbPWB0nX/QQxgH+2vavDK4/mG6zunEl+YrVcVQhcOn6JTrN7ES3Gt34Z+d/Wh0n37SAOMgbHd6gU/VOdJ3VlUvXL1kdR7mxC4kX6PhVR/rW7ss7Xd6xdFT1gnJ6YykR6ZipqdROEUm7HlPlAAAMIElEQVS5NTK7iHwpIicyvdekIHmsJCK83+19utfsTrsv23Em/ozVkZQbOp94no5fdWRQ/UG81fGtQl08wAWNpYwxq281lcLWByYJ+CXTLC9kajq1s4B5LCUiTOw8kSeaPcG9X9yrl3jV75y4coL2X7ZnWMNhvN7h9UJfPMD1jaUGAT8ZY5IKuF239tw9z/FGhzfo8GUHos45d+hFVThsObuFtpFtebrV07za/lWr4ziMqxpL3TIUmHPbtIkisltEPrw1entWRGS0vTlV1KVL7n+OYUSTEUztM5Wes3vqsIge7vsD39Pnv32Y1nca41qNszqOYxlj7vgAVgB7s3j0x9ZZLvO8V+6wnkrAJcD3tmkC+GPbg/l7TnmMMTRv3twUFutPrTeVJlUy761/z2RkZFgdR7lQRkaG+WDDB6bK+1VM1Nkoq+PkCRBlcvG3mON9IMaYLtm9JyIXRKSSMSbmTo2l7IYA3xtj0jKtO8b+NFVEvgCK3Ndc24a2ZfOozQyYN4CdF3Yyve90ivkWszqWcrLU9FSeXfYsa0+tZcOfNhTZTodObyyVyTBuO3zJ1NVOsJ0/2VvAPG6pWqlqrH98PQD3fXGfXqEp4k5ePcl9X9zH+cTz/G/k/4ps8QDXNJZCRMKAasCvty0/294ndw8QDLj/AAj5VMy3GF8P/JoHGzxI6+mtWX5sudWRlBMsPryY1tNbM7ThUL4d8i2lAkpZHcmptLGUBVadWMXwhcMZUn8I/+z8T7cd71LlXnpGOhNWTmDO3jnMHTSXiGoRVkcqEG0s5cY6Ve/EzjE7OXH1BK2nt2b/pf05L6Tc1uG4w7T/sj27Luxi+5jthb545IUWEIuUCyzHt0O+ZVyrcbT/sj2fbf2Mwrg36MluZtzkg40fEDEjgqENhrL04aUEBwZbHcul9BDGDRyOO8zD3z1MkF8Qk3tPdutRuJXNodhDjFw0Eh8vHyL7RVKzbE2rIzmUHsIUIrXL1WbjnzYyoM4A7o28l7+v/jsp6SlWx1JZSLuZxqQNk2gb2ZahDYayevjqIlc88kILiJvw8fJhfJvx7PrzLvZf2k+jyY1YcXyF1bFUJj8f/ZnGUxqz/PhyNo/azNOtn8ZLPPtPSAcUcjNVSlZhwZAFLDm8hCd+fIKWlVsysdNEwsuFWx3NYx2OO8xzy57jUNwhPuz+Ib3DexeJL8I5gmeXTzfWu3Zv9o3dR5OKTbhnxj2MXTKW84nnrY7lUa4kX+H5X54nYkYEHcI6sG/sPvrU7qPFIxMtIG4s0DeQl+97mUPjDhHoG0iDzxrwyqpXtL2mk8UlxfHKqleo9XEtrqVeY9/YfTwf8Tx+3n5WR3M7WkAKgXKB5ZjUbRI7xuzgbMJZwj8O5/U1r+vIZw4WmxTLyytfpvYntbmQeIGoJ6KY1ncaFYIqWB3NbWkBKURCS4XyRf8vWPv4Ws5eO0vtT2ozdslYjl4+anW0Qu10/GleXP4idT6pw+Xky2wbvY3P+31O9TLVrY7m9rSAFEJ1g+vyeb/POfDUAcoWK8s9M+5h0PxBbDyzUW9GyyVjDCuOr2DgvIE0ndqUlPQUdozZwZQ+UwgrHWZ1vEJDbyQrAhJvJBK5I5KPt3yMn7cfI5uM5JHGj+iudxbiU+KZuWsmn0V9hq+XL0+1fIqHG9tu4lP/X25vJNMCUoQYY1h/ej2ROyP5/sD3dKzekZFNRtIzvGehaNTsLMlpySw9spS5++byy7Ff6FGrB+NajuPe0Hv1iko2tIB4uITUBObvm0/kzkgOxx2mT+0+9K/Tn241uxHoG2h1PKdLu5nGiuMrmLN3Dj8e/pFmlZoxrOEw7q93P2WLlbU6ntvTAqJ+c+rqKRYdWsTCQwvZenYrHat3ZECdAfQK71WkDnNOXj3JsqPLWHZsGatPrqZucF2GNRzG4PqDqVSiktXxChWXFBARGQy8DtQDWhljsvyrFpEewH8Ab2C6MebWwEPVgblAWWA78Kgx5kZO29UCkn+Xky+z9MhSFh5cyIrjK6hUohLtQtvRPqw97e5qR9WSVa2OmGsxCTFsObuFlSdWsuzYMq4kX6FbzW50r9mdrjW7UjGootURCy1XFZB6QAYwFXg+qwIiIt7AYWwjlkUDW4Fhxpj9IjIf+M4YM1dEpgC7jDGTc9quFhDHuJlxk90XdrP21FrWnl7L2lNrKeFXgohqETSu0JhG5RvRqEIjqpSoYvm5gqspV9l2bhtbz21ly9ktbD23laS0JFpWbkn7u9rTvVZ3mlRs4vHfTXEUlx7CiMgasi8g9wCvG2O621//zf7WO9hGaa9ojEm/fb470QLiHMYYDsQeYFP0JvZc2MPui7vZc2EP6RnpNCzfkIblG3JXqbuoWrIq1UpVo1rJalQpWcUhd2impqcSmxTLhesXOHr5KEfijnDksv0Rd4SktCSaVmpKy8otaVWlFS0rt6RGmRqWF7aiKrcFxBWn5qsAmUcRjgZaA+WwtYVIzzS9igvyqGyICPVD6lM/pP7vpl9IvMCei3vYf2k/Z+LPsOP8DqKvRXPm2hliEmIoU6wMZQLKEOQXRJBfECX8S9ie+wbh7eVNekb6b4+0jDTSM9JJTksmNimWS0mXiE2KJTU9lZDiIYQEhlCzbE3Cy4ZzX+h9jGw6kvCy4VQMqqjFwg3lWEBEZAWQ1cHkBGPMnUZh/20VWUwzd5ieXY7RwGiA0NCiO8q1O6oQVIEKQRXoUuOPHT5uZtzk4vWLxKfGk3gjkYTUBBJvJNqe30ggw2Tg4+Xzh0cxn2IEBwYTHBhMSPEQSviV0AJRCBWoL0wuRWMbkf2WqsA5IBYoLSI+9r2QW9OzyzENmAa2Q5gCZlIO4u3lTaUSlfQqh4dyxRmnrUC4iFQXET9s7S0X2btfrcbWLxdy7iujlHIzBSogIjJQRKKBe4AlIrLMPr2yiCwFsO9djAOWAQeA+caYffZVvAg8JyJHsZ0TmVGQPEop19IbyZRSf6CDKiulnE4LiFIq37SAKKXyTQuIUirftIAopfKtUF6FEZFLwKlczBqM7Ya1wqwofAYoGp/Dkz7DXcaYkJxmKpQFJLdEJCo3l6LcWVH4DFA0Pod+hj/SQxilVL5pAVFK5VtRLyDTrA7gAEXhM0DR+Bz6GW5TpM+BKKWcq6jvgSilnKhIFRARGSwi+0QkQ0SyPdMsIj1E5JCIHBWRl1yZMSciUlZElovIEfvPMtnMd1NEdtofi1ydMys5/V5FxF9E5tnf3ywiYa5PmbNcfI4RInIp0+9/lBU570REIkXkoojszeZ9EZGP7J9xt4g0y9eGjDFF5oFtdPg6wBqgRTbzeAPHgBqAH7ALqG919kz53gNesj9/CXg3m/kSrc6a198rMBaYYn8+FJhnde58fo4RwCdWZ83hc7QDmgF7s3m/F/ATtpEB2wCb87OdIrUHYow5YIw5lMNsrYCjxpjjxtZCYi7Q3/npcq0/8JX9+VfAAAuz5EVufq+ZP9sCoLO43ziG7v7vI1eMMWuBy3eYpT8w09hswjY6YJ6HlStSBSSXshrk2Z0Gc65gjIkBsP8sn818ASISJSKbRMQdikxufq+/zWNsA03FYxtIyp3k9t/HA/Zd/wUiUi2L992dQ/4OCl3DVCcO8uwyd/oMeVhNqDHmnIjUAFaJyB5jzDHHJMyX3PxeLf/d50JuMv4IzDHGpIrIn7HtVXVyejLHcsh/i0JXQIzzBnl2mTt9BhG5ICKVjDEx9l3Ki9ms45z953F7X56m2I7drZKb3+uteaJFxAcoxZ13s62Q4+cwxsRlevk58K4LcjmaQ/4OPPEQJstBni3OlNkibANMQzYDTYtIGRHxtz8PBtoC+12WMGu5+b1m/myDgFXGfkbPjeT4OW47V9AP21i/hc0i4DH71Zg2QPytQ+c8sfpssYPPPA/EVllTgQvAMvv0ysDS285AH8b2f+wJVue+7TOUA1YCR+w/y9qnt8DWVxggAtiD7QrBHuBPVufO7vcKvAn0sz8PAL4BjgJbgBpWZ87n53gb2Gf//a8G6lqdOYvPMAeIAdLsfxN/Av4M/Nn+vgCf2j/jHrK5apnTQ+9EVUrlmycewiilHEQLiFIq37SAKKXyTQuIUirftIAopfJNC4hSKt+0gCil8k0LiFIq3/4fGHZc4cunHsUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd807a6d0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4lGXa9/HvmQohtJDQibTQBDZINSi9S1VEUBRkERV0UV9dC/rI6rKWB2XXhrSoIEsRFBFQpApIDdJ7h0AoCSEkEEJCrvePGXwiJqTNzD2ZOT/HMUdm7rnLb6I5udtcpxhjUEqpgvCxOoBSqujSAqKUKjAtIEqpAtMCopQqMC0gSqkC0wKilCowLSBKqQLTAqKUKjAtIEqpAvOzOkBBhIaGmurVq1sdQymPtXXr1nhjTFhu8xXJAlK9enViYmKsjqGUxxKRE3mZTw9hlFIFpgVEKVVgWkCUUgWmBUQpVWBaQJRSBeaQAiIi0SJyXkR25/C+iMhHInJYRHaKyF1Z3hsiIofsjyGOyKPc38yZEBoKIrZHaKhtmipaxBEjkolIGyAFmG6MaZjN+z2AZ4EeQEvgP8aYliISAsQAzQADbAWaGmMSb7e9Zs2aGb2MW3R9P+Eo6S++QkJmWS4SQgLluEAYcVTiNFU4RTWSKfWHZcqVg//8Bx55xKLQXkZEthpjmuU2n0PuAzHGrBGR6reZpQ+24mKAjSJSRkQqAe2AZcaYiwAisgzoBsxyRC7lOpkmk6vpV0lOSyblesrvjxvmBn4+fn94TJyyjU9Lf0PoVSiVBpLN+i5SliPU4hAR7KceexMa8NbgRjw2uDaZ+AJaVNyBq24kqwKcyvI61j4tp+l/IiIjgBEA4eHhzkmpsmWM4XTyafZe2MuppFOcunyK2Muxv/+MvRxLyvUUivsVJzggmJKBJQkOCKaEfwn8fPzIyMz4/ZGemc6x9ol08oeEILjmB6FXbY+wK1D+CtRMhIiLiUQkxNDpYgyDrvxfkblCENuJZDMt2JTQklcGt2bw4GqAFhQruKqAZPePjLnN9D9PNGYyMBlshzCOi6aySk1PZdvZbew6t4ud53ay6/wudp/fjZ+PH3eWv5PqZapTtWRVWlRpwQP1H6BqqapULVWV0sVK4yN5O6X2UugX/G/CMMBWQBKKQ3yQ7XE2GI6EwIoa8HkzOBQCGT4QcRGaxEHzM1dpcXo9z5xfz/OZtvUd5w5W045VCe0ZM7QTUEWLiIu4qoDEAtWyvK4KnLFPb3fL9NUuyqSA5LRk1p9az5oTa1hzcg3b4rZRL7QejSs0plH5RvSr349G5RtRIbiCw7bZekw7Hn3pv5S6cZGQjIuEJsdTPvk8lYijJbH05xSBXP99/ovF4UA5+K0SrK8G/24FJ0vDX85C8zPQ9vgJ7j/2FUPTvmJsxpuMGTNWC4iLuKqALASeEZHZ2E6iJhlj4kRkKfAvESlrn68L8KqLMnklYwxb47ayYP8Clh5Zyr4L+2hauSltwtvwZts3aVW1FcEBwU7N0Pf5GlwpX4OnnoKUlD+/L2RSmTPU5jB1OUD91H00it3FQ7E7GLUlAYDLgbC1EmyqChObw2P9IPIsxB45xYkjW7iReRe+Pr5O/RzKcVdhZmHbkwgFzgFvAv4AxpjPRUSAT7CdIL0KPG6MibEvOwx4zb6qccaYL3Lbnl6FyZ/rN67zy/FfWLB/AQsPLiTIP4i+dfvSI6IHLau2pJhfMcuyzZwJo0dDQkJe5jZUJZbmbKElm4hiPS3YTCDXueoPy+/w5/5aT+JTfwWlKp2nV91eDGo4iA41OuDnUyS/N2qZvF6FcUgBcTUtILkzxrD+1Hqit0Xz3f7vqBtalz51+9C3Xl/qhdazOl6ejRwJkyZBZmb27xcjlZZsoj2rKMVlXvafwBdfwL09TzJ/73xm75nN8UvH6V+/P4MaDSKqWlSez9V4My0gXiouOY7pO6YTvT0aQRjWZBiDGw+mcsnKVkdzmJz2WnK6CnPk4hFm757N7D2zSbqWxOORjzOi6QiqlMr2gp9CC4hXMcaw7OgyPt78MetOrqN//f4MazKMVlVbYTt6VDftPLeTSTGTmLV7Fh1rdmRU81G0vaOt/p5uoQXEC6TfSGf27tmM3zCeTJPJ862e56E7H6JEQAmro7m9y2mXmbFjBp9u+RQf8WFU81EMjRxKcf/iVkdzC1pAPNjltMtM2TqFf2/6NxEhEbwU9RLdanfTf0ULwBjDquOr+M+m/7Dl9BZeinqJJ5s9SZB/kNXRLJXXAqJnk4qQ1PRU3lv3HrU+qsWWM1tY8NACVg5ZSfeI7lo8CkhE6FCjA98P/J4ljyxh7cm11PqoFh+s/4Ar169YHc/taQEpAjIyM5iydQoRH0ew5cwW1j6+ltn9Z9O0clOro3mUyIqRfPvQtywdvJQNsRuo9VEtPtzwIddvXM99YS+lBcSNGWP4dt+3NPysIf/d/V/mD5jPvAHzitRl2KKocYXGzBswj2WPLmPlsZU0/Kwhiw4uoige7jubngNxU/su7OPpxU9z6dol3u30Ll1rddXDFIv8eOhHnl/6PNXLVGdC1wnUD6tvdSSn03MgRVRqeiqvr3ydNl+2oX+D/mwdsVVPkFqse0R3dj29i261u9HmyzaM/nE0l9MuWx3LLWgBcSNLDy+l4cSGHEw4yI6ndvBMi2f0+xxuwt/Xn+daPcfekXtJuZ5Co4mNWHZkmdWxLKdfEHADiamJPPPjM2w4tYFPe3xK94juVkdSOQgrEca0PtNYengpw38YTtdaXRnfZTylAkvlvrAH0j0Qi/1y/BciJ0VSrng5do/crcWjiOhauys7n9oJQKOJjfj5yM8WJ7KG7oFYJP1GOmNXjyV6ezTTek+jR0QPqyOpfCpdrDSTe03m5yM/88QPT9Cnbh/GdxlPgG+A1dFcRvdALHAo4RCto1uz/dx2tj+5XYtHEdelVhe2P7mdk0knufeLezlxKU9tZT2CFhAX+37/97SObs1jf3mMRYMWOXSkL2WdssXL8t1D3zGgwQBaTG3B4oOLrY7kEg45hBGRbsB/AF9gqjHm3VvenwC0t78MAsobY8rY37sB7LK/d9IY09sRmdxNpslk3JpxTP5tMoseXkSLKi2sjqQcTET4f1H/j1ZVWzFw/kAGnxzM2x3e9ujBjAp9I5mI+AIHgc7YxjjdAgwyxuzNYf5ngSbGmGH21ynGmHyNoVfUbiRLuZ7C0AVDOZ18mm8HfEulkpWsjqSc7MKVCzz87cP4+fgxt/9cSgaWtDpSvrjyRrIWwGFjzFFjzHVgNrY+MDkZhBf1fTmWeIyoaVGUCizF6iGrtXh4ibASYfz4yI+Elwrn3i/uJfZyrNWRnMIRBSQ/vV3uAGoAK7NMLiYiMSKyUUT6OiCP2/gt7jeioqMYftdwpvWeRqBfoNWRlAv5+fjxec/PebjRw0RNi2LH2R1WR3I4Rxyc5bm3CzAQmGeMuZFlWrgx5oyI1ARWisguY8yRP22kiDWW+uX4Lzz4zYN83vNz7q9/v9VxlEVEhL+3/jvVy1Sn84zOTO83nW61u1kdy2EcsQeSU8+X7AzklsMXY8wZ+8+j2HrCNMluQWPMZGNMM2NMs7CwsMJmdqqFBxbS/5v+zHpglhYPBcCAOwewYOAChi4Yypzdc6yO4zCOKCBbgAgRqSEiAdiKxMJbZxKRukBZYEOWaWVFJND+PBRoDWR78rWomL5jOiN+GMGSh5fQsWZHq+MoNxJVLYpljy7j+aXPM3PnTKvjOEShD2GMMRki8gywFNtl3GhjzB4ReQuIMcbcLCaDgNnmj5d96gOTRCQTWzF7N6erN0XBpJhJjFs7jlVDVnnFV75V/jWq0Ijljy2n84zOZGRmMCRyiNWRCkXHA3GQ6TumM2blGFYPWU2tkFpWx1Fubn/8fjpN78Rb7d9iWJNhVsf5k7xexvXcO1xcaP7e+by8/GVWPrZSi4fKk3qh9Vg5ZCUdp9sOc92xiOSFFpBCWnJoCSOXjGTp4KV62KLypU65Oqx4bAVtv2xLWFAYver2sjpSvul3YQph9fHVDF0wlO8Hfk9kxUir46giqE65Onw/8HuGLRzGxtiNVsfJNy0gBbTvwj4GfDOAOf3n0KpqK6vjqCKsRZUWfNX3K/rO7suB+ANWx8kXLSAFkHA1gV6zevF+5/dpX6N97gsolYseET14t9O7dJvZjbjkOKvj5JkWkHy6fuM698+9nwfqP8DQyKFWx1EeZGjkUIY3GU7PWT1JTU+1Ok6eaAHJB2MMTy96mrLFyvJOp3esjqM80Gv3vkZESAQjl4wsEn1otIDkw783/putcVv5+v6v8RH91SnHExGm9p7KltNbmLR1ktVxcqWXcfNow6kNvPvru2wevpnggHwNX6JUvgQHBPPdQ9/ROro1kRUj3fokvf4zmgeJqYkMmj+IKb2mcEeZO6yOo7xARLkIpvWexoPfPMi5lHNWx8mRFpBcGGMY/sNw+tTtQ++6HjnaonJTver2YshfhvDod4+SaTKtjpMtLSC5mBgzkWOJx3i/8/tWR1FeaGy7sSSlJTFxy0Sro2RLC8ht7Dy3kzdXv8mc/nN0NDFlCT8fP2b0m8HYX8a65U1mWkBykJGZwePfP857nd4jolyE1XGUF6tTrg5j247lsQWPkZGZYXWcP9ACkoMJGyYQUjyExyMftzqKUjzd/GlKB5bmnbXudf+RFpBsHL54mPd+fY9JPSchkt2Qr0q5lo/4EN0nmo83f8zOczutjvM7hxQQEekmIgdE5LCIvJLN+0NF5IKIbLc/hmd5b4iIHLI/LB+eyRjDiB9G8Oo9r1KzbE2r4yj1u6qlqvJW+7cYtWSU29ylWugCYm8s9SnQHWgADBKRBtnMOscYE2l/TLUvGwK8CbTE1l/mTREpW9hMhRG9LZrk68mMbjXayhhKZeuJu57gavpVvt75tdVRAGsaS2XVFVhmjLlojEkElgGWjXmfmJrIqyteZUqvKR7djlAVXb4+vnzW4zNeXv4ySdeSrI7j0sZSD4jIThGZJyI320DkuSmVK4xbO46+9frq4EDKrbWs2pIeET0Yu3qs1VEcUkDy0ljqB6C6MaYxsBz4Kh/L2mYUGWHvYBdz4cKFAofNydHEo3yx/Qveav+Ww9etlKO90/EdZu6aya5zu3Kf2Ylc0ljKGJNgjEmzv5wCNM3rslnW4dTGUq+teI3nWj5HxeCKDl+3Uo4WViKMMfeOYczKMZbmcEljKRHJ2lG6N7DP/nwp0MXeYKos0MU+zaU2xm5k3cl1vHD3C67etFIF9mSzJ9l2dhubYjdZlqHQBcQYkwHcbCy1D5h7s7GUiNz89tnfRGSPiOwA/gYMtS97EXgbWxHaArxln+YyxhheWvYS/+zwT0oElHDlppUqlGJ+xXijzRu8seoNyzJ4fWOp1cdXM+KHEewbtQ9fH1+HrFMpV0m/kU69T+sR3TuattXbOmy9eW0s5fV3ov5r7b945Z5XtHioIsnf1583277JG6vesOTmMq8uIFtOb2F//H4GNx5sdRSlCuyRRo9w/sp5Vh1f5fJte3UBeWfdO7wY9SIBvgFWR1GqwHx9fHkx6kUmbJzg8m17bQHZe2Evv576leF3Dc99ZqXc3CONHmFT7CYOJRxy6Xa9toB8uOFDnm3xLEH+QVZHUarQivsXZ0TTEXy06SOXbtcrC0jStSTm7Z3HE3c9YXUUpRxmZPORzNw1k0vXLrlsm15ZQGbumknnWp2pEFzB6ihKOUzlkpXpHtGdab9Nc9k2va6AGGOYtHUSTzZ90uooSjnc31r8jYkxE112SdfrCsjm05tJuZ5ChxodrI6ilMO1qNICf19/NsRucMn2vK6ATN46mRF3jdDWlMojiQiPNn6UGTtmuGR7XvVXlJqeyvx98xkSafnIiUo5zSONHmHu3rmkZaTlPnMheVUBWXpkKU0rN9Wv7CuPdkeZO2hcoTGLDy12+ra8qoDM2zuP/vX7Wx1DKad7tPGjzNjp/MMYrykgaRlpLD60mH71+1kdRSmne6D+Ayw/upyU6ylO3Y7XFJBlR5fRuEJjPXxRXqF0sdK0rNKS5UeXO3U7XlNA9PBFeZuedXqy+KBzz4O4qrHUCyKy1z4q+woRuSPLezeyNJxaeOuyjpBpMll8aDF96/V1xuqVckv3RdzHksNLnHpTmasaS20DmtlHZZ8HvJ/lvdQsDad64wQ7z+0kpHgI1UpXy31mpTxERLkISviXYPvZ7U7bhksaSxljVhljrtpfbsQ2+rrLrDy2ko41Orpyk0q5hfsi7mPRwUVOW78rG0vd9Ffgxyyvi9n7vWwUkRyPMQrTF2bFsRV667rySt1qd2PZ0WVOW7+rGkvZZhQZDDQD/jfL5HD74K0PA/8WkVrZLVvQvjDpN9JZd3Id7aq3y/MySnmKVlVb8Vvcb6TfSHfK+l3SWApARDoBY4DeWZpMYYw5Y/95FFgNNHFApt9tObOFmmVrEhoU6sjVKlUklC5WmpplazrtPIirGks1ASZhKx7ns0wvKyKB9uehQGtgrwMy/W7tibW0CW/jyFUqVaREVYty2rdzXdVY6n+BYOCbWy7X1gdi7A2nVgHvGmMcWkC2nd1Gs8q5trdQymNFVYti/an1Tlm3nyNWYoxZAiy5Zdr/ZHneKYfl1gONHJEhJ7/F/cbrbV535iaUcmt3V72b11c652/Ao+9ETU5L5nTyaeqF1rM6ilKWqR1SmyvpVzibctbh6/boArLj3A4alm+In49DdrSUKpJEhAZhDdh3YV/uM+eTRxeQbXHbaFLRoRd1lCqS6ofWZ1+8FpB82XFuB3+p8BerYyhlufqh9XUPJL+OJh6ldkhtq2MoZbn6YboHkm/HLx2nepnqVsdQynJ6CJNPGZkZnE4+TXjpcKujKGW5aqWrcenaJZLTkh26Xo8tILGXYylfojyBfoFWR1HKcj7iQ+WSlYlLiXPseh26NjdyLPEYNcrUsDqGUm6jYnBF4pK1gOTJiaQTev5DqSwqBld0+M1kHltA4q/GExaU96/9K+XpKgVX0kOYvLp07RJlipWxOoZSbkP3QPJBC4hSf6R7IPlw6dolyhYva3UMpdxGqcBSDm805dEFRPdAlPo/xfyKkZqe6tB1uqovTKCIzLG/v0lEqmd571X79AMi0tUReQASryVSOrC0o1anVJFXzK8Y1zKuOXSdruoL81cg0RhTG5gAvGdftgG2IRDvBLoBn9nXV2jXb1ynmF8xR6xKKY/glgWEPPSFsb/+yv58HtBRRMQ+fbYxJs0Ycww4bF+fUsrB3LWA5KUvzO/z2MdQTQLK5XFZpZQDpN1IY9vZbQ5dp6v6wuQ0T356yhS4sZRSyvYFU0dzVV+Y3+cRET+gNHAxj8sCBW8spZSyKRlQksiKkQ5dp0v6wthfD7E/7w+sNLaW4QuBgfarNDWACGCzAzIppW5xLeOawy8sFHq0YWNMhojc7AvjC0Tf7AsDxBhjFgLTgBkichjbnsdA+7J7RGQutmZSGcAoY8yNwmYC8PPxIz3TOe38lCqKUjNS3a+AQJ76wlwDHsxh2XHAOEfkyKpMsTJcunbJ0atVqshyxh6Ix96JqgVEqT/SApIPZQK1gCiVVcr1FEr4l3DoOj22gJQtXlYLiFJZnE05S8Xgig5dp8cWkDLFypCYmmh1DKXchhaQfAgpHkJCaoLVMZRyG3EpcVQKruTQdXpsAQkvHc6JpBNWx1DKbegeSD7UKFODY4nHrI6hlNuIS46jUkndA8mT8NLhnE4+7ZT7/5UqaowxnEk+o4cweRXoF0hYUBinL5+2OopSljubcpZAv0CHD/PpsQUEoEbZGhy7pIcxSu2P30/90PoOX69HF5CaZWtyNPGo1TGUsty++H3UC63n8PV6dAFpGNaQned2Wh1DKcvtu7BP90Dyq0mlJg4fgUmpomhf/D7qh2kByZcmFZuw/ex2Mk2m1VGUstTeC3t1DyS/ygWVo0yxMnoeRHm12MuxpGemE1463OHr9ugCAra9kG1xehijvNeGUxuIqhaFrRGCYxWqgIhIiIgsE5FD9p9/usgsIpEiskFE9ojIThF5KMt7X4rIMRHZbn84dsBGbAVka9xWR69WqSJj/an13F31bqesu7B7IK8AK4wxEcAK++tbXQUeM8bcbB71bxHJ2nPyJWNMpP2xvZB5/qR1eGvWnlzr6NUqVWSsj11PVLUop6y7sAUka8Oor4C+t85gjDlojDlkf34GOA+4bFj1qGpR7Di7g+S0ZFdtUim3kZqeyu7zu2lWuZlT1l/YAlLBGBMHYP9Z/nYzi0gLIAA4kmXyOPuhzQQRCSxknj8J8g+ieZXmrDu5ztGrVsrtxZyJ4c6wOwnyD3LK+nMtICKyXER2Z/O4tX1lbuupBMwAHjfm9+uqrwL1gOZACPDybZYvcGOpDtU7sOLYinwto5Qn+PnIz7Sv3t5p68+1gBhjOhljGmbz+B44Zy8MNwvE+ezWISKlgMXA68aYjVnWHWds0oAvuE1f3MI0lupYsyMrj63M1zJKeYLFhxbTs05Pp62/sIcwWRtGDQG+v3UGe7Op74DpxphvbnnvZvERbOdPdhcyT7aaV27OkcQjxF+Nd8bqlXJLZ5LPcCLpBHdXc84VGCh8AXkX6Cwih4DO9teISDMRmWqfZwDQBhiazeXamSKyC9gFhAL/LGSebPn7+tOpZicWH1zsjNUr5ZaWHFpCl1pd8PNxSPunbBVqzcaYBKBjNtNjgOH2518DX+ewfIfCbD8/+tfvz8xdMxkSOST3mZXyAIsOLqJ/g/5O3YbH34l603117mPtybUkXUuyOopSTnct4xqrjq+iW+1uTt2O1xSQUoGlaFe9HT8c/MHqKEo53eKDi2laqSmhQaFO3Y7XFBCwHcbM2zvP6hhKOd2MnTN4tPGjTt+OVxWQXnV7sfLYSi6nXbY6ilJOE381nlXHV/FAgwecvi2vKiBlipWhc63OzNo1y+ooSjnNnN1zuC/iPkoFlnL6tryqgACMuGsEk3+bbHUMpZzGVYcv4IUFpHOtzlxMvUjMmRiroyjlcAfiD3D80nE61+rsku15XQHxER+euOsJJm/VvRDleT7Z/AnDmgxz6s1jWXldAQEY1mQY3+z9Rk+mKo9y6dolZu6ayajmo1y2Ta8sIBWDK9KhRge+3pntDbJKFUnR26LpHtGdKqWquGybXllAAF5o9QLj14/X3rnKI2RkZvDRpo8Y3XK0S7frtQWkdXhrqpWuxpzdc6yOolShLTywkCqlqtCiSo4jYjiF1xYQgNfueY131r2jfWNUkWaMYfz68TzX8jmXb9urC0iXWl0I9Atk0cFFVkdRqsB+OvwTSWlJ3F//fpdv26sLiIjw2j2vMW7tOIwxVsdRKt+MMby+6nXeavcWvj6+Lt++VxcQgH71+5GclsyPh3+0OopS+bZg/wKMMfSr38+S7Tu9sZR9vhtZRiNbmGV6DRHZZF9+jn34Q5fyER/e6/QeLy17Sa/IqCLlRuYN3lj1Bm+3fxsfsWZfwBWNpQBSszSP6p1l+nvABPvyicBfC5mnQHrW6Un5EuWJ3hZtxeaVKpC5e+ZSMrAkPSJ6WJbB6Y2lcmIfSLkDcHOAjnwt70giwvjO43lz9ZvagEoVCVfTr/Lqild5p+M7Tul5m1euaixVzN7TZaOI3CwS5YBLxpibxw2xgOtuobtF08pN6VyzM+//+r5VEZTKs3fWvkOrqq1oV72dpTly/caNiCwHKmbz1ph8bCfcGHNGRGoCK+0jsWf3RZQcL4WIyAhgBEB4eHg+Np13/+zwT5pMasITTZ8gvLRztqFUYR1KOMTEmInseGqH1VFc01jK3hMXY8xRYDXQBIgHyojIzSJWFThzmxwFbiyVV+Glw3mu5XM8vfhpvayr3JIxhtE/jebl1i+79DsvOXFFY6myN3veikgo0BrYa2x/oauA/rdb3tVevudlTiadZPbu2VZHUepPvj/wPccuHWN0K9d+5yUnrmgsVR+IEZEd2ArGu8aYvfb3XgZeEJHD2M6JTCtknkIL8A1gaq+pPL/0ee1kp9xK0rUkRv80mk+6f0KAr8vveMiWFMVd9WbNmpmYGOeOKPb8T8+TkJrA9H7TnbodpfLq8e8fJ8AngEm9Jjl9WyKy1RjTLLf5vP5O1Jy83eFt1p5cy0+Hf7I6ilIs2L+ANSfW8EHXD6yO8gdaQHIQHBDMlF5TGL5wOOevZHtuWCmXOH/lPE8vfprpfacTHBBsdZw/0AJyG51qdmJw48EMXTBUv/KvLGGMYcQPIxjylyG0Dm9tdZw/0QKSi7fbv03itUQmbJhgdRTlhab+NpVjl47xj3b/sDpKtrSA5MLf159ZD8zivV/fY/PpzVbHUV4k5kwMY1aOYU7/OQT6BVodJ1taQPKgepnqfN7zcwbOG0jStSSr4ygvEH81nv5z+/N5z8+pF1rP6jg50gKSR/fXv5+edXoyaP4g/dq/cqqMzAwGzhvIwIYDLRllLD+0gOTDB10+ID0znRd/ftHqKMqDvb7ydcD23Sx3pwUkH/x9/fnmwW/46fBPTIpx/s08yvvM2jWLWbtnMeuBWS7rLlcY7p/QzZQpVoZFDy+idXRraofUpmPNjlZHUh5ixdEVjP5pNCseW0FYCed8YdTRdA+kAGqH1GZO/zk8/O3D7I/fb3Uc5QG2n93OoPmDmPvgXBpVaGR1nDzTAlJA7aq34/1O79NlRheOJR6zOo4qwo5fOk7P//bk0x6fWj5AUH7pIUwhDIkcQsr1FDrN6MSaoWvcYnwGVbQkXE2g29fd+Hvrv/PgnQ9aHSfftIAU0qgWo7iSfoVOMzrxy9BfKF8ip1EdlfqjhKsJdJrRiX71+vG3ln+zOk6B6CGMA/y99d95sMGDdJnRhcTURKvjqCLgwpULdJjegS41u/Cvjv+yOk6BaQFxkH+0+wcdanSg84zOXLhyweo4yo2dSzlH+6/a06tOL97t9K6lo6oXltMbS4lI+yxNpbaLyLWbI7OLyJcicizLe5GFyWMlEeGDLh/QtVZX2nzZhlNJp6yOpNzQ2ZSztP+qPf0b9Oft9m8X6eIBLmgsZYxZdbOpFLY+MFdUYkPTAAAL2klEQVSBn7PM8lKWplPbC5nHUiLCuI7jeOKuJ7jni3v0Eq/6g2OJx2j7ZVsGNRzE2HZji3zxANc3luoP/GiMuVrI7bq1F+5+gX+0+wftvmxHzBnnDr2oiobNpzfTOro1z7Z4ljfavmF1HIdxVWOpmwYCs26ZNk5EdorIhJujt2dHREbYm1PFXLjg/ucYhkYOZVLPSXSf2V2HRfRy3+37jp7/7cnkXpN5psUzVsdxLGPMbR/AcmB3No8+2DrLZZ038TbrqQRcAPxvmSZAILY9mP/JLY8xhqZNm5qiYt2JdabS+Erm/XXvm8zMTKvjKBfKzMw0H67/0FT5oIqJOR1jdZx8AWJMHv4Wc70PxBjTKaf3ROSciFQyxsTdrrGU3QDgO2NMepZ1x9mfponIF4DHfc21dXhrNg3fRN85fdl+bjtTe02luH9xq2MpJ0vLSOP5pc+z5sQa1v91vcd2OnR6Y6ksBnHL4UuWrnaC7fzJ7kLmcUvVSldj3ePrALj3i3v1Co2HO37pOPd+cS9nU87y67BfPbZ4gGsaSyEi1YFqwC+3LD/T3id3FxAKuP8ACAVU3L84X/f7mofufIiWU1uy7MgyqyMpJ1h0cBEtp7ZkYMOBzB8wn9LFSlsdyam0sZQFVh5byZAFQxjQYAD/6vgvtx3vUuVdRmYGY1aMYdbuWczuP5uoalFWRyoUbSzlxjrU6MD2J7dz7NIxWk5tyd4Le3NfSLmtgwkHaftlW3ac28FvT/5W5ItHfmgBsUi5oHLMHzCfZ1o8Q9sv2/LZls8oinuD3uxG5g0+3PAhUdOiGHjnQJY8soTQoFCrY7mUHsK4gYMJB3nk20cIDghm4n0T3XoUbmVzIP4AwxYOw8/Hj+je0dQKqWV1JIfSQ5gipE65Omz46wb61u3LPdH38D+r/odrGdesjqWykX4jnfHrx9M6ujUD7xzIqiGrPK545IcWEDfh5+PH6Faj2fHUDvZe2EujiY1YfnS51bFUFj8d/onGnzdm2dFlbBq+iWdbPouPePefkA4o5GaqlKrCvAHzWHxwMU/88ATNKzdnXIdxRJSLsDqa1zqYcJAXlr7AgYQDTOg6gfsi7vOIL8I5gneXTzd2X5372DNyD5EVI7l72t2MXDySsylnrY7lVRJTE3nx5xeJmhZFu+rt2DNyDz3r9NTikYUWEDcW5B/Ea/e+xoFnDhDkH8Sdn93J6ytf1/aaTpZwNYHXV75O7Y9rczntMntG7uHFqBcJ8A2wOprb0QJSBJQLKsf4LuPZ9uQ2TiefJuLjCMauHqsjnzlY/NV4XlvxGnU+qcO5lHPEPBHD5F6TqRBcwepobksLSBESXjqcL/p8wZrH13D68mnqfFKHkYtHcvjiYaujFWknk07y8rKXqftJXS6mXmTriK1M6T2FGmVrWB3N7WkBKYLqhdZjSu8p7Bu1j5DiIdw97W76z+3PhlMb9Ga0PDLGsPzocvrN6UeTSU24lnGNbU9u4/Oen1O9THWr4xUZeiOZB0i5nkL0tmg+3vwxAb4BDIscxuDGg3XXOxtJ15KYvmM6n8V8hr+PP6Oaj+KRxrab+NT/yeuNZFpAPIgxhnUn1xG9PZrv9n1H+xrtGRY5jO4R3YtEo2ZnSU1PZcmhJczeM5ufj/xMt9rdeKb5M9wTfo9eUcmBFhAvl5yWzNw9c4neHs3BhIP0rNOTPnX70KVWF4L8g6yO53TpN9JZfnQ5s3bP4oeDP3BXpbsY1HAQ99e/n5DiIVbHc3taQNTvTlw6wcIDC1lwYAFbTm+hfY329K3blx4RPTzqMOf4peMsPbyUpUeWsur4KuqF1mNQw0E82OBBKpWsZHW8IsUlBUREHgTGAvWBFsaYbP+qRaQb8B/AF5hqjLk58FANYDYQAvwGPGqMuZ7bdrWAFNzF1IssObSEBfsXsPzociqVrESb8Da0rd6WNne0oWqpqlZHzLO45Dg2n97MimMrWHpkKYmpiXSp1YWutbrSuVZnKgZXtDpikeWqAlIfyAQmAS9mV0BExBc4iG3EslhgCzDIGLNXROYC3xpjZovI58AOY8zE3LarBcQxbmTeYOe5naw5sYY1J9ew5sQaSgaUJKpaFI0rNKZR+UY0qtCIKiWrWH6u4NK1S2w9s5UtZ7aw+fRmtpzZwtX0qzSv3Jy2d7Sla+2uRFaM9PrvpjiKSw9hRGQ1OReQu4Gxxpiu9tev2t96F9so7RWNMRm3znc7WkCcwxjDvvh9bIzdyK5zu9h5fie7zu0iIzODhuUb0rB8Q+4ofQdVS1WlWulqVCtVjSqlqjjkDs20jDTir8Zz7so5Dl88zKGEQxy6aH8kHOJq+lWaVGpC88rNaVGlBc0rN6dm2ZqWFzZPldcC4opT81WArKMIxwItgXLY2kJkZJlexQV5VA5EhAZhDWgQ1uAP08+lnGPX+V3svbCXU0mn2HZ2G7GXYzl1+RRxyXGULV6WssXKEhwQTHBAMCUDS9qe+wfj6+NLRmbG74/0zHQyMjNITU8l/mo8F65eIP5qPGkZaYSVCCMsKIxaIbWICIng3vB7GdZkGBEhEVQMrqjFwg3lWkBEZDmQ3cHkGGPM7UZh/30V2Uwzt5meU44RwAiA8HDPHeXaHVUIrkCF4Ap0qvnnDh83Mm9w/sp5ktKSSLmeQnJaMinXU2zPryeTaTLx8/H706O4X3FCg0IJDQolrEQYJQNKaoEoggrVFyaPYrGNyH5TVeAMEA+UERE/+17Izek55ZgMTAbbIUwhMykH8fXxpVLJSnqVw0u54ozTFiBCRGqISAC29pYL7d2vVmHrlwu595VRSrmZQhUQEeknIrHA3cBiEVlqn15ZRJYA2PcungGWAvuAucaYPfZVvAy8ICKHsZ0TmVaYPEop19IbyZRSf6KDKiulnE4LiFKqwLSAKKUKTAuIUqrAtIAopQqsSF6FEZELwIk8zBqK7Ya1oswTPgN4xufwps9whzEmLLeZimQBySsRicnLpSh35gmfATzjc+hn+DM9hFFKFZgWEKVUgXl6AZlsdQAH8ITPAJ7xOfQz3MKjz4EopZzL0/dAlFJO5FEFREQeFJE9IpIpIjmeaRaRbiJyQEQOi8grrsyYGxEJEZFlInLI/rNsDvPdEJHt9sdCV+fMTm6/VxEJFJE59vc3iUh116fMXR4+x1ARuZDl9z/cipy3IyLRInJeRHbn8L6IyEf2z7hTRO4q0IaMMR7zwDY6fF1gNdAsh3l8gSNATSAA2AE0sDp7lnzvA6/Yn78CvJfDfClWZ83v7xUYCXxufz4QmGN17gJ+jqHAJ1ZnzeVztAHuAnbn8H4P4EdsIwO2AjYVZDsetQdijNlnjDmQy2wtgMPGmKPG1kJiNtDH+enyrA/wlf35V0BfC7PkR15+r1k/2zygo7jfOIbu/v9Hnhhj1gAXbzNLH2C6sdmIbXTAfA8r51EFJI+yG+TZnQZzrmCMiQOw/yyfw3zFRCRGRDaKiDsUmbz8Xn+fx9gGmkrCNpCUO8nr/x8P2Hf954lItWzed3cO+Tsocg1TnTjIs8vc7jPkYzXhxpgzIlITWCkiu4wxRxyTsEDy8nu1/HefB3nJ+AMwyxiTJiJPYdur6uD0ZI7lkP8WRa6AGOcN8uwyt/sMInJORCoZY+Lsu5Tnc1jHGfvPo/a+PE2wHbtbJS+/15vzxIqIH1Ca2+9mWyHXz2GMScjycgrwngtyOZpD/g688RAm20GeLc6U1UJsA0xDDgNNi0hZEQm0Pw8FWgN7XZYwe3n5vWb9bP2BlcZ+Rs+N5Po5bjlX0BvbWL9FzULgMfvVmFZA0s1D53yx+myxg88898NWWdOAc8BS+/TKwJJbzkAfxPYv9hirc9/yGcoBK4BD9p8h9unNsPUVBogCdmG7QrAL+KvVuXP6vQJvAb3tz4sB3wCHgc1ATaszF/BzvAPssf/+VwH1rM6czWeYBcQB6fa/ib8CTwFP2d8X4FP7Z9xFDlctc3vonahKqQLzxkMYpZSDaAFRShWYFhClVIFpAVFKFZgWEKVUgWkBUUoVmBYQpVSBaQFRShXY/weQ5Eg4wPPa9gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd807a1b908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize predictions\n",
    "\n",
    "sum_error = 0\n",
    "for i in range(num_test):\n",
    "    y = ((data_test[i,:,:]-np.array(mc_estimation)[i,:,:])**2)\n",
    "    sum_error = sum_error +y\n",
    "#     print(((data_test[i,:,:]-np.array(mc_estimation)[i,:,:])**2).mean())\n",
    "\n",
    "\n",
    "# print(sum_error/num_test)    \n",
    "\n",
    "k=93\n",
    "# print(mc_cost)\n",
    "# estimation_error_k = np.linalg.norm(data_test[k,:,:]-np.array(mc_estimation)[k,:,:],2)\n",
    "estimation_error_k = np.mean((data_test[k,:,:]-np.array(mc_estimation)[k,:,:])**2)\n",
    "print('Estimation error for test point k=',k,':', estimation_error_k)\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(np.array(mc_estimation)[k,:,0],np.array(mc_estimation)[k,:,1],color='b', marker='o')\n",
    "plt.plot(data_test[k,:,0],data_test[k,:,1],color='r', linewidth=4)\n",
    "t = np.linspace(0,np.pi*2,100)\n",
    "plt.plot(np.cos(t), np.sin(t), linewidth=1, color='g')\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(np.array(est_data_batch)[k,:,0],np.array(est_data_batch)[k,:,1],color='b', marker='o')\n",
    "plt.plot(data_train_batch[k,:,0],data_train_batch[k,:,1],color='r', linewidth=4)\n",
    "t = np.linspace(0,np.pi*2,100)\n",
    "plt.plot(np.cos(t), np.sin(t), linewidth=1, color='g')\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9923916\n"
     ]
    }
   ],
   "source": [
    "# print(np.array(mc_estimation)[0,:,0])\n",
    "# print(data_test[0,:,0])\n",
    "# print(mc_cost)\n",
    "\n",
    "print(np.mean(squared_error))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Can an RNN learn to follow a circle? New cost function\n",
    "\n",
    "Input to RNN: starting point\n",
    "output of RNN: float for next number\n",
    "Cost: (e^(j*output) - real_data)^2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "1600000\n",
      "1700000\n",
      "1800000\n",
      "1900000\n",
      "2000000\n",
      "2100000\n",
      "2200000\n",
      "2300000\n",
      "2400000\n",
      "2500000\n",
      "2600000\n",
      "2700000\n",
      "2800000\n",
      "2900000\n",
      "3000000\n",
      "3100000\n",
      "3200000\n",
      "3300000\n",
      "3400000\n",
      "3500000\n",
      "3600000\n",
      "3700000\n",
      "3800000\n",
      "3900000\n",
      "4000000\n",
      "4100000\n",
      "4200000\n",
      "4300000\n",
      "4400000\n",
      "4500000\n",
      "4600000\n",
      "4700000\n",
      "4800000\n",
      "4900000\n",
      "Data generation complete.\n"
     ]
    }
   ],
   "source": [
    "# create training data\n",
    "\n",
    "import numpy as np\n",
    "import scipy.signal as sig\n",
    "from numpy import linalg as LA\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create training and test data for the NN\n",
    "\n",
    "epochs = 5000\n",
    "batch_size = 1000\n",
    "\n",
    "num_train=epochs*batch_size\n",
    "num_test=batch_size\n",
    "\n",
    "data_length = 20\n",
    "omega = 1/100 # rate of rotation, number of rotations / symbol\n",
    "\n",
    "\n",
    "# QPSK\n",
    "data_train = np.zeros((num_train, data_length,2))\n",
    "data_test = np.zeros((num_test, data_length,2))\n",
    "start_train = np.zeros((num_train, 1))\n",
    "start_test = np.zeros((num_test, 1))\n",
    "\n",
    "for i in range(num_train):\n",
    "    \n",
    "    # first choose a random starting point on the unit circle\n",
    "    # there are 2pi radians in a circle, choose random number between 0 and 2pi\n",
    "    random_start = np.random.rand()*2*np.pi\n",
    "#     random_start = 0\n",
    "    start_train[i] = random_start\n",
    "    # real part of data\n",
    "    data_train[i,0,:]=np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_train[i,j,0]=(data_train[i,j-1,0]*np.cos(omega)-data_train[i,j-1,1]*np.sin(omega))\n",
    "        data_train[i,j,1]=(data_train[i,j-1,0]*np.sin(omega)+data_train[i,j-1,1]*np.cos(omega))\n",
    "        \n",
    "    if i % 100000 == 0:\n",
    "        print(i)\n",
    "    \n",
    "\n",
    "for i in range(num_test):\n",
    "    # first choose a random starting point on the unit circle\n",
    "    # there are 2pi radians in a circle, choose random number between 0 and 2pi\n",
    "    random_start = np.random.rand()*2*np.pi\n",
    "#     random_start = 0 \n",
    "    start_test[i]=random_start\n",
    "    # real part of data\n",
    "    data_test[i,0,:]=np.array([np.cos(random_start), np.sin(random_start)])\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_test[i,j,0]=(data_test[i,j-1,0]*np.cos(omega)-data_test[i,j-1,1]*np.sin(omega))\n",
    "        data_test[i,j,1]=(data_test[i,j-1,0]*np.sin(omega)+data_test[i,j-1,1]*np.cos(omega))\n",
    "\n",
    "print(\"Data generation complete.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import linalg as LA\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# fix random seed for reproducibility\n",
    "np.random.seed()\n",
    "\n",
    "learning_rate = 0.01\n",
    "decay = 1 - 10*1e-10\n",
    "\n",
    "state_size = 1 # state size\n",
    "output_size = 1 # size of the output\n",
    "\n",
    "batch_data_placeholder = tf.placeholder(tf.float32, [None, data_length,2])\n",
    "# batch_start_placeholder = tf.placeholder(tf.float32, [None, 1])\n",
    "\n",
    "init_state = tf.placeholder(tf.float32, [None, state_size])\n",
    "\n",
    "# Unpack columns\n",
    "data = tf.unstack(batch_data_placeholder, axis=1)\n",
    "\n",
    "# Forward pass\n",
    "current_state = init_state\n",
    "est_data_series = []\n",
    "for current_data in data:\n",
    "    \n",
    "#     input_start_state_concatenated = tf.concat([batch_start_placeholder, current_state],1) \n",
    "    \n",
    "#     layer_1 = tf.layers.dense(\n",
    "#       current_state, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_2 = tf.layers.dense(\n",
    "#       layer_1, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_3 = tf.layers.dense(\n",
    "#       layer_2, 100, tf.nn.sigmoid, use_bias=True)\n",
    "#     layer_4 = tf.layers.dense(\n",
    "#       layer_3, 100, tf.nn.sigmoid, use_bias=True)\n",
    "    \n",
    "    \n",
    "#     layer_5 = tf.layers.dense(\n",
    "#       layer_4, state_size, activation=tf.identity, use_bias=True)\n",
    "\n",
    "#     layer_3 = tf.layers.dense(\n",
    "#         layer_1, state_size, activation=tf.identity, use_bias=True)\n",
    "\n",
    "    next_state = layer_5\n",
    "\n",
    "    # take cos(next_state) and sin(next_state) as x and y coordinates\n",
    "    x = tf.cos(next_state)\n",
    "    y = tf.sin(next_state)\n",
    "    app = tf.concat([x,y],axis=1)\n",
    "    est_data_series.append(app)\n",
    "#     print(est_data_series)\n",
    "#     print(np.array(est_data_series).shape)\n",
    "    current_state = next_state\n",
    "\n",
    "# est_data_series = (np.transpose(np.array(est_data_series), axes=(1, 0, 2))).tolist()    \n",
    "\n",
    "est_data_series=tf.transpose(est_data_series, perm=[1,0,2])\n",
    "# estimation_cost = tf.losses.mean_squared_error(data,est_data_series)\n",
    "squared_difference = tf.squared_difference(batch_data_placeholder, est_data_series)\n",
    "\n",
    "estimation_cost = tf.reduce_mean(tf.squared_difference(batch_data_placeholder, est_data_series))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(estimation_cost)\n",
    "                                                                                  \n",
    "# finally setup the initialisation operator\n",
    "init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Train Cost 1.0003148317337036, Test Cost: 0.9941355586051941\n",
      "Epoch 100, Train Cost 0.8524659872055054, Test Cost: 0.9991777539253235\n",
      "Epoch 200, Train Cost 0.7977465987205505, Test Cost: 0.9997045993804932\n",
      "Epoch 300, Train Cost 0.8013573288917542, Test Cost: 1.015687108039856\n",
      "Epoch 400, Train Cost 0.7743804454803467, Test Cost: 1.0024611949920654\n",
      "Epoch 500, Train Cost 0.7695026397705078, Test Cost: 1.0078575611114502\n",
      "Epoch 600, Train Cost 0.7582810521125793, Test Cost: 1.0010544061660767\n",
      "Epoch 700, Train Cost 0.763292133808136, Test Cost: 0.9994281530380249\n",
      "Epoch 800, Train Cost 0.7561063766479492, Test Cost: 1.009779930114746\n",
      "Epoch 900, Train Cost 0.7652855515480042, Test Cost: 0.9945132732391357\n",
      "Epoch 1000, Train Cost 0.7632288336753845, Test Cost: 1.0047279596328735\n",
      "Epoch 1100, Train Cost 0.7639355659484863, Test Cost: 1.001190423965454\n",
      "Epoch 1200, Train Cost 0.7419256567955017, Test Cost: 0.9892204999923706\n",
      "Epoch 1300, Train Cost 0.7452003955841064, Test Cost: 0.988985538482666\n",
      "Epoch 1400, Train Cost 0.7550675272941589, Test Cost: 0.9931908249855042\n",
      "Epoch 1500, Train Cost 0.754633367061615, Test Cost: 1.0056124925613403\n",
      "Epoch 1600, Train Cost 0.7613629698753357, Test Cost: 1.0022894144058228\n",
      "Epoch 1700, Train Cost 0.7263949513435364, Test Cost: 0.9922515749931335\n",
      "Epoch 1800, Train Cost 0.7630087733268738, Test Cost: 1.0108039379119873\n",
      "Epoch 1900, Train Cost 0.7301133871078491, Test Cost: 1.0120538473129272\n",
      "Epoch 2000, Train Cost 0.7539327144622803, Test Cost: 0.98954176902771\n",
      "Epoch 2100, Train Cost 0.754536509513855, Test Cost: 0.996286928653717\n",
      "Epoch 2200, Train Cost 0.7386767864227295, Test Cost: 0.9901115298271179\n",
      "Epoch 2300, Train Cost 0.750716507434845, Test Cost: 0.9899276494979858\n",
      "Epoch 2400, Train Cost 0.766841471195221, Test Cost: 1.0117255449295044\n",
      "Epoch 2500, Train Cost 0.7505195140838623, Test Cost: 1.006944179534912\n",
      "Epoch 2600, Train Cost 0.7483825087547302, Test Cost: 1.0050874948501587\n",
      "Epoch 2700, Train Cost 0.7501927614212036, Test Cost: 0.9871960878372192\n",
      "Epoch 2800, Train Cost 0.7523463368415833, Test Cost: 1.0060508251190186\n",
      "Epoch 2900, Train Cost 0.7718337774276733, Test Cost: 1.0045101642608643\n",
      "Epoch 3000, Train Cost 0.7525781989097595, Test Cost: 1.002107858657837\n",
      "Epoch 3100, Train Cost 0.7271500825881958, Test Cost: 1.0012140274047852\n",
      "Epoch 3200, Train Cost 0.7315848469734192, Test Cost: 0.9982950091362\n",
      "Epoch 3300, Train Cost 0.7359769344329834, Test Cost: 1.0009890794754028\n",
      "Epoch 3400, Train Cost 0.7731338739395142, Test Cost: 1.0053952932357788\n",
      "Epoch 3500, Train Cost 0.7618325352668762, Test Cost: 1.000978708267212\n",
      "Epoch 3600, Train Cost 0.7510561347007751, Test Cost: 0.9990511536598206\n",
      "Epoch 3700, Train Cost 0.7387593984603882, Test Cost: 0.9993584156036377\n",
      "Epoch 3800, Train Cost 0.7492571473121643, Test Cost: 1.0070537328720093\n",
      "Epoch 3900, Train Cost 0.7633465528488159, Test Cost: 0.9978363513946533\n",
      "Epoch 4000, Train Cost 0.7656594514846802, Test Cost: 1.0067967176437378\n",
      "Epoch 4100, Train Cost 0.7475017309188843, Test Cost: 1.00285804271698\n",
      "Epoch 4200, Train Cost 0.7496377825737, Test Cost: 1.0006749629974365\n",
      "Epoch 4300, Train Cost 0.7396652102470398, Test Cost: 1.0092166662216187\n",
      "Epoch 4400, Train Cost 0.7268354296684265, Test Cost: 0.9971376061439514\n",
      "Epoch 4500, Train Cost 0.7602371573448181, Test Cost: 0.9950052499771118\n",
      "Epoch 4600, Train Cost 0.7579121589660645, Test Cost: 1.009820580482483\n",
      "Epoch 4700, Train Cost 0.7508755922317505, Test Cost: 0.9995996356010437\n",
      "Epoch 4800, Train Cost 0.7496947050094604, Test Cost: 0.9933921694755554\n",
      "Epoch 4900, Train Cost 0.7483022212982178, Test Cost: 1.0029418468475342\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEWCAYAAABBvWFzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XuYXFWd7vHvmxsmXEJuYCQmjYfgNRq1RcfxHqKiYnjmKDC2GB087WXGgfHoCBMvg5pzos4z4h1bFINEDXgZAuJgyIiIB8GOBpLoQKImISZCFIhokFt+54+9Klaaquqq7l21q7rez/PU07vWXnvXWtVV+1d7rbXXVkRgZmY2WuOKLoCZmY0NDihmZpYLBxQzM8uFA4qZmeXCAcXMzHLhgGJmZrlwQDHLiaSQdFxavkDS+4ouUxEkzZX0R0njiy6LtZYDihVG0jZJd0g6tCztzZKuLXsekjZKGleW9mFJX66yzxdJ2p8OaOWPv2pmXYaKiLdGxIda+ZrtIiJ2RMRhEfFw0WWx1nJAsaJNAM4aJs9jgNMb2OeudEArf9ww8iK2B//it3bngGJF+xjwLklH1sjzUeA8SRNG+2KSjpX0A0n3Slor6dOSLknrXiRp55D82ySdmJZPkHSDpHsk7U7bTqryOl+W9OG0fMWQs6X9kt6Y1j0hleMuSbdKOnXIPj4n6SpJfwJeXEf9rpX0IUk/SnX8nqSZZetfLWlzqsO1kp5YY18h6a2Stki6W9JnJCmtGyfpvZK2S7pT0sWSpqZ1PWnbCen5GyX9KpXn15L6yl7j7yT9Iu3/aknzhqujtS8HFCvaIHAt8K4aeb4F/AF4Yw6v91VgPTAT+BCwtIFtHwb+KW37V8Ai4O3DbRQRJ5fOlIDXAL8F1qWmvrWpTEcBfwt8VtKTyzZ/HbAcOBy4XtLrJN0yzEu+DnhT2uck0nsr6Xjga8DZwCzgKuCKakExeRXwLOBpwKnAy1L6G9PjxcDjgMOATw/dONXxk8BJEXE48FxgQ1p3CvAvwN+k8vwwlc86lAOKtYP3A++QNKvK+gDeB7xf0iF17O8x6Rd4+eNQSXPJDo7vi4j7I+I64Ip6CxkR6yPixxHxUERsAz4PvLDe7dMB/WLgtIi4nexgvS0iLkr7/CnwTbKgU3J5RPwoIvZHxJ8j4qsR8dRhXuqiiLgtIu4DLgUWpvTTgO9ExNqIeBD4N2Ay2UG+mhURcU9E7AC+X7avPuDfI+JXEfFH4Fzg9CpnkfuBp0iaHBG7I2JzSn8L8H8j4hcR8RDwf4CFPkvpXA4oVriI2ARcCZxTI89VwA6gv45d7oqII4c8/kTWF3N3Wi7ZXm85JR0v6UpJv5X0B7ID4MzhtkvbTgUuJwtmP0zJ84Bnlwc+sgP1o8s2vb3e8pX5bdnyPrKzB8jqf6C+EbE/7f+Y0e4rLU8Aji7fOL3XpwFvBXZL+o6kJ6TV84BPlNX9LkDDlMfamAOKtYsPAP+L2geT9wLLgCkjfI3dwLTyUWXA3LLlP5XvO3WCl581fQ74b2B+RBxB1lyj4V40jVD7KvD9iPh82arbgR8MCXyHRcTbyvLkOR34LrKDeKlcAh4L/Ga0+yJ7Hx8C7hiaMSKujojFwGyy9+8LadXtwFuG1H9yRPy/EZTH2oADirWFiNgKrAb+sUaea4GNNNbvUb79drI+m/MkTZL0PODksiy3AY+S9EpJE8kCWHkT2+FkfTl/TL+yyw/8tSwHDuWRo9muBI6XdIakienxrFod5aN0KfBKSYtS/f43cD8wkgP414B/SoMcDiM7W1udmq4OkHR0GghwaHqtP5L1RQFcAJxb6jOSNFXSa0dUM2sLDijWTj5IduCt5b3A9GHyPEaPvA7lf6Z1rwOeTda88gGyPg0AImIvWSf7hWS/2v8ElI/6elfa/l6yX9mr66pV1tn+HODusvL0RcS9wEvJhkTvImte+ggHB7GDSOqTtLna+loi4lbg9cCngN+RBdOTI+KBEezuS8BXgOuAXwN/Bt5RId84ssC1i+w9fyFpIENEfJusvl9PTYibgJNGUBZrE/INtqybSfpX4LiIeH3RZTHrdD5DMTOzXDigmJlZLtzkZWZmufAZipmZ5WLUcyN1kpkzZ0ZPT0/RxTAz6yjr16//XURUm8nigK4KKD09PQwODhZdDDOzjiKprhkl3ORlZma5cEAxM7NcOKCYmVkuHFDMzCwXDihmZpaLQgOKpJen255ulfSIe2FIOkTS6rT+Rkk9ZevOTem3SnrZ0G3zsmrjKnrO72HceePoOb+HVRtXNeulzMw6WmHDhtO9Jj4DLCab0fUnktZExM/Lsp1JdkOk4ySdTjYz6WmSnkQ2Q+uTyW70c42k4yPiYXK0auMq+q/oZ9+D+wDYvnc7/Vdk93fqW9BXa1Mzs65T5BnKCcDWdAvRB4CvA0uG5FkCrEzL3wAWpZsCLQG+nm7j+mtga9pfrpatW3YgmJTse3Afy9Yty/ulzMw6XpEB5RgOvr3pTh55t74DedKNe/YCM+rcFgBJ/ZIGJQ3u2bOnoQLu2LujoXQzs25WZECpdOvUoTNVVstTz7ZZYsRARPRGRO+sWcPOHHCQuVPnNpRuZtbNigwoO8nuZ10yh+yubhXzSJoATCW761s9247a8kXLmTLx4NuXT5k4heWLluf9UmZmHa/IgPITYH66J/Uksk72NUPyrOEv9w9/DfBfkc23vwY4PY0COxaYD9yUdwH7FvQxcPIA86bOQ4h5U+cxcPKAO+TNzCoobJRXRDwk6R+Aq4HxwJciYrOkDwKDEbEG+CLwFUlbyc5MTk/bbpZ0KfBz4CHg7/Me4VXSt6DPAcTMrA5ddYOt3t7e8GzDZmaNkbQ+InqHy+cr5c3MLBcOKGZmlgsHFDMzy4UDipmZ5cIBxczMcuGAYmZmuXBAMTOzXDigmJlZLhxQzMwsFw4oZmaWCwcUMzPLhQOKmZnlwgHFzMxy4YBiZma5cEAxM7NcOKCYmVkuHFDMzCwXDihmZpYLBxQzM8uFA4qZmeXCAcXMzHLhgGJmZrkoJKBImi5praQt6e+0KvmWpjxbJC0tS79W0q2SNqTHUa0rvZmZVVLUGco5wLqImA+sS88PImk68AHg2cAJwAeGBJ6+iFiYHne2otBmZlZdUQFlCbAyLa8ETqmQ52XA2oi4KyLuBtYCL29R+czMrEFFBZSjI2I3QPpbqcnqGOD2suc7U1rJRam5632SVO2FJPVLGpQ0uGfPnjzKbmZmFUxo1o4lXQM8usKqZfXuokJapL99EfEbSYcD3wTOAC6utJOIGAAGAHp7e6NSHjMzG72mBZSIOLHaOkl3SJodEbslzQYq9YHsBF5U9nwOcG3a92/S33slfZWsj6ViQDEzs9YoqslrDVAatbUUuLxCnquBl0qaljrjXwpcLWmCpJkAkiYCrwI2taDMZmZWQ1EBZQWwWNIWYHF6jqReSRcCRMRdwIeAn6THB1PaIWSB5RZgA/Ab4Autr4KZmZVTRPd0K/T29sbg4GDRxTAz6yiS1kdE73D5fKW8mZnlwgHFzMxy4YBiZma5cEAxM7NcOKCYmVkuHFDMzCwXDihmZpYLBxQzM8uFA4qZmeXCAcXMzHLhgGJmZrlwQDEzs1w4oJiZWS4cUMzMLBcOKGZmlgsHFDMzy4UDipmZ5cIBxczMcuGAYmZmuXBAMTOzXDigmJlZLhxQzMwsF4UEFEnTJa2VtCX9nVYl339KukfSlUPSj5V0Y9p+taRJrSm5mZlVU9QZyjnAuoiYD6xLzyv5GHBGhfSPAB9P298NnNmUUpqZWd2KCihLgJVpeSVwSqVMEbEOuLc8TZKAlwDfGG57MzNrnaICytERsRsg/T2qgW1nAPdExEPp+U7gmGqZJfVLGpQ0uGfPnhEX2MzMapvQrB1LugZ4dIVVy0a76wppUS1zRAwAAwC9vb1V85mZ2eg0LaBExInV1km6Q9LsiNgtaTZwZwO7/h1wpKQJ6SxlDrBrlMU1M7NRKqrJaw2wNC0vBS6vd8OICOD7wGtGsr2ZmTVHUQFlBbBY0hZgcXqOpF5JF5YySfohcBmwSNJOSS9Lq94DvFPSVrI+lS+2tPRmZvYITWvyqiUifg8sqpA+CLy57Pnzq2z/K+CEphXQzMwa5ivlzcwsFw4oZmaWi2EDiqS59aSZmVl3q+cM5T/qTDMzsy5WtVNe0vHAE4Gpkl5dtuoI4FHNLpiZmXWWWqO8ngz8DXAk8Nqy9HuBtzSzUGZm1nmqBpSI+DbwbUnPi4jrW1gmMzPrQPX0obxS0hGSJki6Ok2b8rqml8zMzDpKPQHlpIj4A/Aqsjm3nkJ2pbqZmdkB9QSUienvK4CvRcQeaszua2Zm3ameqVeukrQJeBj4e0kzgfubWywzM+s0w56hRMS7ye6Q+MyIeBC4j2z0l5mZ2QHDnqFImkA2VfwLsrvv8gPgC00ul5mZdZh6mrw+AxwKfCk9fz3wdKC/WYUyM7POU09AeU5EPK3s+fck3dysApmZWWeqZ5TXfkk9pSdpeX9zimNmZp2qnjOUfwauk3QbIOA44MymlsrMzDrOsAElItZKejzZRJECfh4R9zW9ZGZm1lFqzTb8t8D4iLgkBZCfpvQ3S7o3Ila3qpBmZtb+avWhvBtYUyH9MrJmMDMzswNqBZQJaQ6vg0TEXv4yHYuZmRlQO6BMkjRlaKKkw4BDmlckMzPrRLUCypeAyyTNKSWk5a8CF43mRSVNl7RW0pb0d1qVfP8p6R5JVw5J/7KkX0vakB4LR1MeMzMbvaoBJSI+CnwXuDHdA+UO4MfA9yJixShf9xxgXUTMB9al55V8DDijyrp3R8TC9NgwyvKYmdko1bywMSI+HRHHAI8HnhARcyLi0zm87hJgZVpeCZxS5fXXkd1y2MzM2lw9V8oTEfdExN05vu7REbE77Xs3cNQI9rFc0i2SPi6pap+OpH5Jg5IG9+zZM9LympnZMOoKKCMh6RpJmyo8luSw+3OBJwDPAqZT4w6SETEQEb0R0Ttr1qwcXtrMzCqpa/r6iHhouLShIuLEGvu8Q9LsiNgtaTbZrYXrVjq7Ae6XdBHwrka2NzOz/NVzhnJTnWmNWAMsTctLgcsb2TgFIZTdoOUUYNMoy9OwVRtX0XN+D+POG0fP+T2s2riq1UUwM2srtaZeOQqYDUyWtIBsHi+AI4BHXJ/SoBXApZLOBHYAr02v2Qu8NSLenJ7/kKxp6zBJO4EzI+JqYJWkWalMG4C3jrI8DVm1cRX9V/Sz78F9AGzfu53+K7Lbw/Qt6GtlUczM2oYiovIK6U3A3wELgZ/xl4ByL3BRRFzWkhLmqLe3NwYHB0e9n57ze9i+d/sj0udNnce2s7eNev9mZu1E0vqI6B0uX9UzlIi4CLhI0qkRcWmupetwO/buaCjdzKwb1NOHcpSkIwAkXSDpJkmLmlyutjZ36tyG0s3MukE9AaU/Iv4g6aXAHOBtwEebW6z2tnzRcqZMPLgbacrEKSxftLygEpmZFa+egFLqZDmJrO9kfZ3bjVl9C/oYOHmAeVPnIcS8qfMYOHnAHfJm1tWqdsofyCBdDMwEjgeeShZMrouIZzS/ePnKq1PezKybjLpTvsybgGcCWyNin6SZ+J7yZmY2xLBNVxHxMPA4sr4TgMn1bGdmZt1l2MAg6dPAi4HXp6Q/ARc0s1BmZtZ56mnyem5EPEPSzwAi4i5Jk5pcLjMz6zD1NF09KGkcabSXpBnA/qaWyszMOk49AeUzwDeBWZLOA64HPtLUUpmZWcepNTnkhIh4KCIulrQeOJFsPq/XRkTLZ/c1M7P2VqsP5SbgGQARsRnY3JISmZlZR6rV5KUa68zMzA5S6wxllqR3VlsZEf/ehPKYmVmHqhVQxgOH4TMVMzOrQ62AsjsiPtiykpiZWUdzH4qZmeWiVkDp6ptomZlZY6oGlIi4q5UFMTOzzuZZg83MLBcOKGZmlouqAUXSYyV9XdIPJf2LpIll6/5jNC8qabqktZK2pL/TKuRZKOkGSZsl3SLptLJ1x0q6MW2/2rMfm5kVr9YZypeAa4F3ALOBH6SZhgHmjfJ1zwHWRcR8YF16PtQ+4A0R8WTg5cD5ko5M6z4CfDxtfze+g6SZWeFqBZRZEXFBRGyIiHcAnwWuk/Q/SFPZj8ISYGVaXgmcMjRDRNwWEVvS8i7gTrKr9wW8BPhGre3NzKy1al3YOFHSoyLizwARcYmk3wJXA4eO8nWPjojdab+7JR1VK7OkE4BJwC+BGcA9EfFQWr0TOKbGtv1AP8DcuXNHWWwzM6um1hnKhcCzyxMi4hrgtcCw09dLukbSpgqPJY0UUNJs4CvAmyJiP5UvuKx6xhQRAxHRGxG9s2bNauSlzcysAVXPUCLi41XSfybpO8PtOCJOrLZO0h2SZqezk9lkzVmV8h0BfAd4b0T8OCX/DjiydL8WYA6wa7jymJlZc4102HDVWYjrtAZYmpaXApcPzZBGbn0buDgiLiulR0QA3wdeU2t7MzNrrZEGlNHO87UCWCxpC7A4PUdSr6QLU55TgRcAb5S0IT0WpnXvAd4paStZn8oXR1keMzMbJWU/+BvcSNoRER3Xw93b2xuDg4NFF8PMrKNIWh8RvcPlq3VP+Xup3NktYPIoymZmZmNQrU75w1tZEDMz62yey8vMzHLhgGJmZrlwQGmhVRtX0XN+D+POG0fP+T2s2riq6CKZmeWm1tQrlqNVG1fRf0U/+x7cB8D2vdvpv6IfgL4FfUUWzcwsFz5DaZFl65YdCCYl+x7cx7J1ywoqkZlZvhxQclatWWvH3h0V81dLNzPrNG7yylGtZq25U+eyfe/2R2wzd2rHXR9qZlaRz1ByVKtZa/mi5UyZOOWgdVMmTmH5ouWtLKKZWdM4oOSoVrNW34I+Bk4eYN7UeQgxb+o8Bk4ecIe8mY0ZbvLK0XDNWn0L+hxAzGzM8hlKjtysZWbdzAElR27WMrNuNqLp6zuVp683M2tcvdPX+wzFzMxy4YBiZma5cEAxM7NcOKCYmVkuHFDMzCwXDihmZpYLBxQzM8tFIQFF0nRJayVtSX+nVcizUNINkjZLukXSaWXrvizp15I2pMfC1tbAzMyGKuoM5RxgXUTMB9al50PtA94QEU8GXg6cL+nIsvXvjoiF6bGh+UU2M7NaigooS4CVaXklcMrQDBFxW0RsScu7gDuBWS0roZmZNaSogHJ0ROwGSH+PqpVZ0gnAJOCXZcnLU1PYxyUdUmPbfkmDkgb37NmTR9lzV+0uj2ZmnaRpAUXSNZI2VXgsaXA/s4GvAG+KiP0p+VzgCcCzgOnAe6ptHxEDEdEbEb2zZrXfCU7pLo/b924niAN3eXRQsbHGP5zGvqYFlIg4MSKeUuFxOXBHChSlgHFnpX1IOgL4DvDeiPhx2b53R+Z+4CLghGbVo9lq3eXRLG9FHdT9w6k7FNXktQZYmpaXApcPzSBpEvBt4OKIuGzIulIwEln/y6amlraJat3l0SxPRR7U/cOpOxQVUFYAiyVtARan50jqlXRhynMq8ALgjRWGB6+StBHYCMwEPtza4uendDfHSuluIrA8FXlQ9w+n7lDILYAj4vfAogrpg8Cb0/IlwCVVtn9JUwvYQssXLaf/iv6DvuhTJk7hFfNfcVB66dck4Bt22YgUeVAf7vbYNjb4SvmCVbvL41VbrnITwRhV1JlnrbPhZvPtsbuDA0ob6FvQx7azt7H/A/vZdvY2+hb0uYlgjCqyH6PIg3o73x7bTcv58S2A21TP+T0VmwjmTZ3HtrO35fY6qzauYtm6ZezYu4O5U+eyfNHytviSj1Wt+r9W4//3wUoBfmiTc7sEu3bhWwB3uJH+mmzk11atX8vV9uNfc6NT9JlnpbPhblDtc+vRZ/kqpFPehlf6ojfya3Lor63hOvKrfZnO+u5Z3PfQfY/Yz492/IiVN6/sqIEC7faL3J3Tj9Ts/1Gt70XRAX6scZPXGNJoc8q488YR1P//H6/xPBwP173/vDV64GnH5ox2LFORWvF+1PpeAIU2QXYKN3l1oUZ/bTX6q7hSMKm1/zyNpDO7HZsz2rlzugit+B/V+l4UPfpsrDUhO6B0oGofwkaHhVb7Ms2YPKNi/vEa39D+8zSSA0+7Nmd0az9GJa34H9X6XhQZ4MfidDQOKB2m1oew0V9b1b5MnzjpExX30//M/sJ+zY3kwFPkdRdWn1b8j4b7XhQV4Gv9SOrUMxcHlA5T60NY69dWtQ9opS9Ttf189pWfLezX3EgOPEU3Z9jwWvE/atdmxmo/hko/EjvxzMWd8h2mWke6EPs/sL/CFu3bEVytk71SOjCiOuQ1gqjdRouNJd363lYbLFD04JdK6u2Ud0DpMCO5MK7oi+kqqRbklj5t6UFDk0vpAycPAI0Nox5pufIKZma1VPsODG2BKKn1o7HZHFAqGAsBZSRnGyM5q2m2dvx1Vu29nTxhMr+/7/eFlGmkRjLEuhvPEopW6X1ftm5Z2/0A9LDhMWok7cHt2Dldrf24yKHJ1fqnKgWTVpVpJBodPdTOo406tXO6XpX6MGv1K7X7++GA0oEaHZXSjp3T1YJZkUOTGw0Qw5Wp0S//SA4WlbZpdIh1O16vA+0b6Jp9UK/2oxFoeKqkVnOTV5dotyaNkfShNLu81ZrhZkyecdBUNPWUqdGmyeHyN9K302gbfDs2icLI+v5aPY0LdObns1HuQ6mgmwNKO2pklFcrLzTLY0BAowfDWvmr3YStWt9Oo/1Q7ThoAxoPdHkf7Nutf6PRqZJKn508vksOKBU4oNhw8gpmjR4Ma+WvNqFkLUPPVEZzdlSUPINyowf7dhyBVa1+tTTyOajFnfLWNtqlfbceeV013ehAiFrpjfbtlNrc6x240a4X/jXa95fnNC7V+pWK7OMbyVRJre4bc0CxpmrXjtVma/RgWCt/tYPVjMkzqm7TaGBsx/nFGg10eY5mrDUKsd3uelltqqQiRkw6oFhTtesIomZr9GBYK3+1YPOJkz7RlmcWeWok0OU5mrFaEGr07C9vjUyVVJqef6hmnk25D8VyU6n/4YxvndGWI4g6TbuN0mtXeU610479So3Isw5t3ykvaTqwGugBtgGnRsTdQ/LMA74FjAcmAp+KiAvSumcCXwYmA1cBZ8UwlXFAaZ6xdJW5GYyNIJ5XHTohoHwUuCsiVkg6B5gWEe8ZkmdSKuP9kg4DNgHPjYhdkm4CzgJ+TBZQPhkR3631mg4ozVPkGHnrPu14sG/HMuWlE0Z5LQFWpuWVwClDM0TEAxFxf3p6CKm8kmYDR0TEDems5OJK21vrVOvou+u+u8Z8O38njWIbC9pxoEc7lqkIRZ6h3BMRR5Y9vzsiplXI91jgO8BxwLsj4jOSeoEVEXFiyvN84D0R8aoK2/cD/QBz58595vbtjY3jtvq068VxzTYW2to7TTt+1tqxTHlqizMUSddI2lThsaTefUTE7RHxVLKAslTS0YAqZa2y/UBE9EZE76xZs0ZWERtWO84X1grdOoqtlmafsbXjrZ3bsUxFmNDMnZfOICqRdIek2RGxOzVh3TnMvnZJ2gw8H/gRMKds9RxgVx5ltpEp/Rofq23I1fhAcrChZ2ylph8gt89CtZkDipw9ux3LVIQi+1DWAEvT8lLg8qEZJM2RNDktTwP+Grg1InYD90p6jiQBb6i0vbVWO14c12zteGuAIrXijK0dz4bbsUxFKDKgrAAWS9oCLE7PkdQr6cKU54nAjZJuBn4A/FtEbEzr3gZcCGwFfgnUHOFl1gw+kBysFWds7ThVTDuWqQi+sNFslMbycNFa2m02Xmuetr8OpQgOKFbSrUEgL+14PxtrnrYY5WXWjnzNwOhV6yu5astVbvrpYj5Dsa4z1q8ZyJvnaDOfoVhXaeTaBw/1rV+1s7npk6dXzN+to9tq6aaZFBxQrOM12oTlob71q9a0BXh0Wx26rXnVAcU6XqPXPniob/26eY62PHTbTApNvVLerBUabcLq1qv6R6LWFeClmztZdd3WvOqAYh1vJNNe+GBYn+WLllccHuyzufp025QsbvKyjucmrObxFeCj022fTQ8btjHBFypauxoLn01fKV+BA4qZWeN8HYqZmbWUA4qZmeXCAcXMzHLhgGJmZrlwQDEzs1x01SgvSXuAR15lVJ+ZwO9yLE6ncL27S7fWG7q37vXUe15EzBpuR10VUEZD0mA9w+bGGte7u3RrvaF7655nvd3kZWZmuXBAMTOzXDig1G+g6AIUxPXuLt1ab+jeuudWb/ehmJlZLnyGYmZmuXBAMTOzXDig1EHSyyXdKmmrpHOKLs9oSfqSpDslbSpLmy5praQt6e+0lC5Jn0x1v0XSM8q2WZryb5G0tIi61EvSYyV9X9IvJG2WdFZKH9P1BpD0KEk3Sbo51f28lH6spBtTPVZLmpTSD0nPt6b1PWX7Ojel3yrpZcXUqH6Sxkv6maQr0/MxX2cASdskbZS0QdJgSmv+Zz0i/KjxAMYDvwQeB0wCbgaeVHS5RlmnFwDPADaVpX0UOCctnwN8JC2/AvguIOA5wI0pfTrwq/R3WlqeVnTdatR5NvCMtHw4cBvwpLFe71RmAYel5YnAjalOlwKnp/QLgLel5bcDF6Tl04HVaflJ6fN/CHBs+l6ML7p+w9T9ncBXgSvT8zFf51TubcDMIWlN/6z7DGV4JwBbI+JXEfEA8HVgScFlGpWIuA64a0jyEmBlWl4JnFKWfnFkfgwcKWk28DJgbUTcFRF3A2uBlze/9CMTEbsj4qdp+V7gF8AxjPF6A6Q6/DE9nZgeAbwE+EZKH1r30nvyDWCRJKX0r0fE/RHxa2Ar2fejLUmaA7wSuDA9F2O8zsNo+mfdAWV4xwC3lz3fmdLGmqMjYjdkB1/gqJRerf4d+76k5oynk/1S74p6p6afDcCdZAeGXwL3RMRDKUt5PQ7UMa3fC8yg8+p+PvDPwP70fAZjv84lAXxP0npJ/Smt6Z/1CTkUfKxThbRuGmtdrf4d+b5IOgz4JnB2RPwh+xFaOWuFtI6td0Q8DCyUdCTwbeCJlbKlvx1fd0mvAu6MiPWSXlRKrpB1zNR5iL+OiF2SjgLWSvrvGnlzq7shqUdoAAAEh0lEQVTPUIa3E3hs2fM5wK6CytJMd6TTXNLfO1N6tfp33PsiaSJZMFkVEd9KyWO+3uUi4h7gWrK28iMllX5UltfjQB3T+qlkTaSdVPe/Bl4taRtZM/VLyM5YxnKdD4iIXenvnWQ/IE6gBZ91B5Th/QSYn0aHTCLrsFtTcJmaYQ1QGsWxFLi8LP0NaSTIc4C96XT5auClkqal0SIvTWltKbWHfxH4RUT8e9mqMV1vAEmz0pkJkiYDJ5L1IX0feE3KNrTupffkNcB/RdZLuwY4PY2IOhaYD9zUmlo0JiLOjYg5EdFD9p39r4joYwzXuUTSoZIOLy2TfUY30YrPetGjETrhQTYK4jaydudlRZcnh/p8DdgNPEj2K+RMsvbidcCW9Hd6yivgM6nuG4Hesv38HVkn5VbgTUXXa5g6P4/sdP0WYEN6vGKs1zuV96nAz1LdNwHvT+mPIzs4bgUuAw5J6Y9Kz7em9Y8r29ey9J7cCpxUdN3qrP+L+MsorzFf51THm9Njc+mY1YrPuqdeMTOzXLjJy8zMcuGAYmZmuXBAMTOzXDigmJlZLhxQzMwsFw4o1nUkzUizsG6Q9FtJvyl7PqnOfVwk6fHD5Pl7SX05lfn6NNttqZyr89hv2f53lq5VMRspDxu2ribpX4E/RsS/DUkX2fdjf8UNW0zS9cA/RMSGJu1/J/CUyK6kNxsRn6GYJZKOk7RJ0gXAT4HZkgYkDSq7j8j7y/JeL2mhpAmS7pG0Qtn9Rm5I8ych6cOSzi7Lv0LZfUlulfTclH6opG+mbb+WXmthA2W+RNLnJP1Q0m2STkrpkyWtVHZPjJ9KekFKnyDp46met0h6e9nuzlZ275BbJB0/6jfUuo4DitnBngR8MSKeHhG/Ibt/RC/wNGCxpCdV2GYq8IOIeBpwA9nVxZUoIk4A3g2UgtM7gN+mbVeQzYJczeqyJq8VZemPBV4InAwMSDoE+EfggYhYAJwBfCU1570NeAzwtIh4Ktk8VyV3RMTTyaZ7f2eNcphV5NmGzQ72y4j4Sdnzv5V0Jtl35TFkAefnQ7a5LyK+m5bXA8+vsu9vleXpScvPAz4CEBE3S9pco2ynVWnyujQ1zd0q6Xay+aaeB3ws7XezpF3AcWTzeJ0f2ezDRET5fXHKy/eKGuUwq8gBxexgfyotSJoPnAWcEBH3SLqEbM6noR4oW36Y6t+r+yvkqTp/fgOGdoRWm3q89HrVOk4rlc+sbm7yMqvuCOBe4A/6yx3s8nY9cCqApAVkZ0CNem2aKfZ4suavLcB1QF/a7xPJboG8Ffge8DZJ49O66aOugVniXyFm1f2UrHlrE9n9tH/UhNf4FHCxpFvS620iu1tgJasl3ZeW74iIUoDbShZAjgL6I+IBSZ8CPi9pI9ms0m9I6Z8naxK7RdJDwOfI7q1uNmoeNmxWIGU3c5oQEX9OTWzfA+bHX25TO9z2lwDfiIj/aGY5zerhMxSzYh0GrEuBRcBb6g0mZu3GZyhmZpYLd8qbmVkuHFDMzCwXDihmZpYLBxQzM8uFA4qZmeXi/wNJ2Y2WAW13vgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7e16d0978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "\n",
    "# start the session\n",
    "with tf.Session() as sess:\n",
    "    #    initialize the variables\n",
    "    sess.run(init_op)\n",
    "    mc_losses = []\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        \n",
    "        current_start = epoch*batch_size\n",
    "        data_train_batch = data_train[current_start:(current_start+batch_size),:,:]\n",
    "        start_train_batch = start_train[current_start:(current_start+batch_size)]\n",
    "        \n",
    "        begin_state = start_train_batch\n",
    "\n",
    "        _,cost,est_data_batch = sess.run([optimizer, estimation_cost, est_data_series], \n",
    "                                          feed_dict={batch_data_placeholder: data_train_batch, \n",
    "                                                     init_state: begin_state})    \n",
    "\n",
    "        if epoch % 100 == 0: \n",
    "            \n",
    "            begin_state = np.random.rand(batch_size,state_size)\n",
    "            mc_cost, mc_estimation = sess.run([estimation_cost, est_data_series], \n",
    "                                             feed_dict={batch_data_placeholder: data_test, \n",
    "                                                        init_state: begin_state})\n",
    "            \n",
    "            \n",
    "#             plt.plot(epoch, mc_cost, 'bo')\n",
    "#             plt.plot(epoch, zero_force_error, 'ro')\n",
    "#             plt.plot(epoch, mse_error, 'go')\n",
    "            print('Epoch {}, Train Cost {}, Test Cost: {}'.format(epoch, cost, mc_cost))\n",
    "            plt.plot(epoch,np.log(cost),'go')\n",
    "#             print('Epoch {}, Train Cost {}'.format(epoch, cost))\n",
    "    \n",
    "    \n",
    "    plt.xlabel('Training Epoch')\n",
    "    plt.ylabel('L2 Test Cost')\n",
    "    plt.title('NN Equalizer: no noise')\n",
    "    # plt.text(1000, .025, r'NN equalizer')\n",
    "    # plt.text(0.5, .025, r'Zero Force equalizer')\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimation error for test point k= 97 : nan\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4lGX2//H3SaOFntCJgRA6LEg1KL1LVUCwgSziWnZd/em6tpXVZS1fLGujRwVZiqCIgEaqgNRQQ+89BAJJSEwhIffvjxnciAlpM/NMZs7ruubK5JmnfGY0h6fNfcQYg1JKFYWP1QGUUiWXFhClVJFpAVFKFZkWEKVUkWkBUUoVmRYQpVSRaQFRShWZFhClVJFpAVFKFZmf1QGKIigoyISGhlodQymPtX379nhjTHB+85XIAhIaGkp0dLTVMZTyWCJyqiDz6SGMUqrItIAopYpMC4hSqsi0gCilikwLiFKqyBxSQEQkUkQuisjePF4XEflQRI6KyB4RuT3Ha6NF5Ij9MdoReZRSruGoPZDPgb63eL0fEG5/jAcmA4hIFeA1oAPQHnhNRCo7KJNSyskcch+IMWadiITeYpbBwCxjGz9xs4hUEpGaQFdghTHmCoCIrMBWiOY6IpdynWyTTWpmKskZyaRcS/n1cd1cx8/H73ePMn5lCCobRIVSFRARq+OrInLVjWS1gTM5fj9rn5bX9N8RkfHY9l4ICQlxTkqVK2MM55LPsf/Sfs4kneHM1TOcvXr2159nr54l5VoKZfzKEBgQSPlS5QkMCKScfzn8fPzIys769ZGZnUlWdhapmalcTr1MelY6QWWDCCobRHC5YKqVq0b9SvUJrxpOeJVwwquGE1w2WIuMm3JVAcntv765xfTfTzRmGjANoG3btjoStJOkZaax88JOYuJi2BO3h5iLMey9uBc/Hz+aVWtGaKVQ6pSvQ/va7bm3yb3UqVCHOhXqULF0RXyk8EfE6VnpXE69THxqPPGp8VxIucCxhGOsOrGKKdFTOHLlCFnZWYRXCad1jda0q92O9rXb0yy4Gf6+/k74BFRhuKqAnAXq5vi9DnDePr3rTdPXuiiTApIzktl4ZiPrTq1j3el17IzdSeOgxrSs3pIW1VowtMlQWlRrQfXA6k7Zfmm/0tSuUJvaFXLd8QTgStoVDsUfYkfsDjae2cgHmz/gdNJp/lDjD7Sr1Y4ut3WhR/0eVChVwSkZVd7EUW0d7OdAlhpjmufy2t3AU0B/bCdMPzTGtLefRN0O3LgqswNoc+OcSF7atm1r9LswRWOMYXvsdhYfXEzUsSgOXDpAm1pt6BzSmS6hXehYpyOBAYFWx8zX1YyrbD+/nS3ntrD6xGo2nd1Eqxqt6BPWhz5hfbi95u34+vhaHbPEEpHtxpi2+c7niAIiInOx7UkEAXHYrqz4AxhjpojtAPZjbCdIU4FHjDHR9mXHAi/ZVzXRGPNZftvTAlI4165f46eTP7H44GKWHF5CWf+yDGk0hP7h/elQpwOl/UpbHbHYUjNTWXdqHVFHo4g6FsXFXy4ysNFARjUfRfd63fHzKZHfG7WMSwuIq2kByZ8xho1nNhK5M5JvDn5Do6BGDG40mCGNh9A4qLHV8ZzudNJpFu1fxLx98ziZeJJhTYYxqsUoIupGFOlcjbfRAuKlYpNjmbV7FpG7IhGEsa3H8mDLB6lVvpbV0Sxz7Mox5u2dx7x980hKT+KRVo8wvs34W5538XZaQLyIMYYVx1fw0daP2HB6A8OaDGNs67F0rNNRL3/eZE/cHqZGT2Xu3rn0qN+DJ9s9SZfbuujndBMtIF4g83om8/bOY9KmSWSbbJ7p+Az3NbuPcgHlrI7m9q5mXGX27tl8su0TfMSHJ9s9yZhWYyjjX8bqaG5BC4gHu5pxlenbp/PBlg8IrxLO8xHP07dBX/1XtAiMMaw5uYb/bPkP285t4/mI53ms7WOU9S9rdTRLFbSA6NmkEiQtM423N7xN2IdhbDu/jcX3LWb16NX0C++nxaOIRITu9brz7chvWf7ActafXk/Yh2G8u/Fdfrn2i9Xx3J4WkBIgKzuL6dunE/5RONvOb2P9I+uZN2webWq1sTqaR2lVoxVf3/c1UQ9GsensJsI+DOO9Te9x7fo1q6O5LS0gbswYw9cHvqb5p835797/smjEIhaOWOgVl2Gt1LJ6SxaOWMiKh1aw+sRqmn/anKWHl1ISD/edTc+BuKkDlw7w+LLHSUxP5K2eb9EnrI8epljk+yPf80zUM4RWCuX9Pu/TJLiJ1ZGcTs+BlFBpmWm8svoVOn/emWFNh7F9/HY9QWqxfuH9iHk8hr4N+tL58848/f3TXM24anUst6AFxI1EHY2i+eTmHL58mN1/2s1T7Z/S73O4CX9ff/7a8a/sf2I/KddSaDG5BSuOrbA6luX0CwJuICEtgae+f4pNZzbxSf9P6Bfez+pIKg/B5YKZOXgmUUejGPfdOPqE9WFS70le+01g3QOx2E8nf6LV1FZULVOVvU/s1eJRQvRp0Ic9f9oDQIvJLfjx2I8WJ7KG7oFYJPN6JhPWTiByVyQzB82kf3h/qyOpQqpYuiLTBk7jx2M/8uh3jzK40WAm9Z5EgG+A1dFcRvdALHDk8hE6RXZiV9wudj22S4tHCdc7rDe7HtvF6aTT3PXZXZxKLFBbWY+gBcTFvj34LZ0iO/HwHx5m6ailThvpS7lW5TKV+ea+bxjRdATtZ7Rn2eFlVkdyCYccwohIX+A/gC8wwxjz1k2vvw90s/9aFqhmjKlkf+06EGN/7bQxZpAjMrmbbJPNxHUTmbZjGkvvX0r72u2tjqQcTET4fxH/j451OjJy0UgePP0gb3R/w6MHMyr2jWQi4gscBnphG+N0GzDKGLM/j/n/DLQ2xoy1/55ijCnUGHol7UaylGspjFk8hnPJ5/h6xNfULF/T6kjKyS79con7v74fPx8/FgxbQPlS5a2OVCiuvJGsPXDUGHPcGHMNmIetD0xeRuFFfV9OJJwgYmYEFUpVYO3otVo8vERwuWC+f+B7QiqEcNdnd3H26lmrIzmFIwpIYXq73AbUA1bnmFxaRKJFZLOIDHFAHrexI3YHEZERjLt9HDMHzaSUXymrIykX8vPxY8qAKdzf4n4iZkaw+8JuqyM5nCMOzgrc2wUYCSw0xlzPMS3EGHNeROoDq0Ukxhhz7HcbKWGNpX46+RPDvxrOlAFTuKfJPVbHURYREf7W6W+EVgql1+xezBo6i74NbtUFtmRxxB5IXj1fcjOSmw5fjDHn7T+PY+sJ0zq3BY0x04wxbY0xbYODg4ub2amWHFrCsK+GMffeuVo8FAAjmo1g8cjFjFk8hvl751sdx2EcUUC2AeEiUk9EArAViSU3zyQijYDKwKYc0yqLSCn78yCgE5DrydeSYtbuWYz/bjzL719Oj/o9rI6j3EhE3QhWPLSCZ6KeYc6eOVbHcYhiH8IYY7JE5CkgCttl3EhjzD4ReR2INsbcKCajgHnmt5d9mgBTRSQbWzF7K6+rNyXB1OipTFw/kTWj13jFV75V4bWo3oKVD6+k1+xeZGVnMbrVaKsjFYuOB+Igs3bP4uXVL7N29FrCqoRZHUe5uYPxB+k5qyevd3udsa3HWh3ndwp6Gddz73BxoUX7F/HCyhdY/fBqLR6qQBoHNWb16NX0mGU7zHXHIlIQWkCKafmR5Tyx/AmiHozSwxZVKA2rNmTVw6vo8nkXgssGM7DRQKsjFZp+F6YY1p5cy5jFY/h25Le0qtHK6jiqBGpYtSHfjvyWsUvGsvnsZqvjFJoWkCI6cOkAI74awfxh8+lYp6PVcVQJ1r52e74Y8gVD5g3hUPwhq+MUihaQIricepmBcwfyTq936FavW/4LKJWP/uH9eavnW/Sd05fY5Fir4xSYFpBCunb9GvcsuId7m9zLmFZjrI6jPMiYVmMY13ocA+YOIC0zzeo4BaIFpBCMMTy+9HEql67Mmz3ftDqO8kAv3fUS4VXCeWL5EyWiD40WkEL4YPMHbI/dzpf3fImP6EenHE9EmDFoBtvObWPq9qlWx8mXXsYtoE1nNvHWz2+xddxWAgMKNXyJUoUSGBDIN/d9Q6fITrSq0cqtT9LrP6MFkJCWwKhFo5g+cDq3VbrN6jjKC4RXDWfmoJkM/2o4cSlxVsfJkxaQfBhjGPfdOAY3GsygRh452qJyUwMbDWT0H0bz0DcPkW2yrY6TKy0g+ZgcPZkTCSd4p9c7VkdRXmhC1wkkZSQxedtkq6PkSgvILeyJ28Nra19j/rD5OpqYsoSfjx+zh85mwk8T3PImMy0gecjKzuKRbx/h7Z5vE1413Oo4yos1rNqQCV0m8PDih8nKzrI6zm9oAcnD+5vep0qZKjzS6hGroyjF4+0ep2Kpiry53r3uP9ICkoujV47y9s9vM3XAVERyG/JVKdfyER8iB0fy0daP2BO3x+o4v3JIARGRviJySESOisjfc3l9jIhcEpFd9se4HK+NFpEj9oflwzMZYxj/3XhevPNF6leub3UcpX5Vp0IdXu/2Ok8uf9Jt7lItdgGxN5b6BOgHNAVGiUjTXGadb4xpZX/MsC9bBXgN6ICtv8xrIlK5uJmKI3JnJMnXknm649NWxlAqV4/e/iipmal8uedLq6MA1jSWyqkPsMIYc8UYkwCsACwb8z4hLYEXV73I9IHTPbodoSq5fH18+bT/p7yw8gWS0pOsjuPSxlL3isgeEVkoIjfaQBS4KZUrTFw/kSGNh+jgQMqtdajTgf7h/ZmwdoLVURxSQArSWOo7INQY0xJYCXxRiGVtM4qMt3ewi7506VKRw+bleMJxPtv1Ga93e93h61bK0d7s8SZzYuYQExeT/8xO5JLGUsaYy8aYDPuv04E2BV02xzqc2ljqpVUv8dcOf6VGYA2Hr1spRwsuF8zLd73My6tftjSHSxpLiUjOjtKDgAP251FAb3uDqcpAb/s0l9p8djMbTm/g2TuedfWmlSqyx9o+xs4LO9lydotlGYpdQIwxWcCNxlIHgAU3GkuJyI1vn/1FRPaJyG7gL8AY+7JXgDewFaFtwOv2aS5jjOH5Fc/zr+7/olxAOVduWqliKe1Xmlc7v8qra161LIPXN5Zae3It478bz4EnD+Dr4+uQdSrlKpnXM2n8SWMiB0XSJbSLw9Zb0MZSXn8n6r/X/5u/3/l3LR6qRPL39ee1Lq/x6ppXLbm5zKsLyLZz2zgYf5AHWz5odRSliuyBFg9w8ZeLrDm5xuXb9uoC8uaGN3ku4jkCfAOsjqJUkfn6+PJcxHO8v/l9l2/bawvI/kv7+fnMz4y7fVz+Myvl5h5o8QBbzm7hyOUjLt2u1xaQ9za9x5/b/5my/mWtjqJUsZXxL8P4NuP5cMuHLt2uVxaQpPQkFu5fyKO3P2p1FKUc5ol2TzAnZg6J6Yku26ZXFpA5MXPoFdaL6oHVrY6ilMPUKl+LfuH9mLljpsu26XUFxBjD1O1TeazNY1ZHUcrh/tL+L0yOnuyyS7peV0C2nttKyrUUutfrbnUUpRyufe32+Pv6s+nsJpdsz+sKyLTt0xh/+3htTak8kojwUMuHmL17tku251V/RWmZaSw6sIjRrSwfOVEpp3mgxQMs2L+AjKyM/GcuJq8qIFHHomhTq41+ZV95tNsq3UbL6i1ZdmSZ07flVQVk4f6FDGsyzOoYSjndQy0fYvYe5x/GeE0BycjKYNmRZQxtMtTqKEo53b1N7mXl8ZWkXEtx6na8poCsOL6CltVb6uGL8goVS1ekQ+0OrDy+0qnb8ZoCoocvytsMaDiAZYedex7EVY2lnhWR/fZR2VeJyG05Xrueo+HUkpuXdYRsk82yI8sY0niIM1avlFu6O/xulh9d7tSbylzVWGon0NY+KvtC4J0cr6XlaDg1CCfYE7eHKmWqULdi3fxnVspDhFcNp5x/OXZd2OW0bbiksZQxZo0xJtX+62Zso6+7zOoTq+lRr4crN6mUW7g7/G6WHl7qtPW7srHUDX8Evs/xe2l7v5fNIpLnMUZx+sKsOrFKb11XXqlvg76sOL7Caet3VWMp24wiDwJtgf/LMTnEPnjr/cAHIhKW27JF7QuTeT2TDac30DW0a4GXUcpTdKzTkR2xO8i8numU9buksRSAiPQEXgYG5WgyhTHmvP3ncWAt0NoBmX617fw26leuT1DZIEeuVqkSoWLpitSvXN9p50Fc1ViqNTAVW/G4mGN6ZREpZX8eBHQC9jsg06/Wn1pP55DOjlylUiVKRN0Ip30711WNpf4PCAS+uulybRMg2t5wag3wljHGoQVk54WdtK2Vb3sLpTxWRN0INp7Z6JR1+zliJcaY5cDym6b9I8fznnkstxFo4YgMedkRu4NXOr/izE0o5dbuqHMHr6x2zt+AR9+JmpyRzLnkczQOamx1FKUs06BKA37J/IULKRccvm6PLiC743bTvFpz/HwcsqOlVIkkIjQNbsqBSwfyn7mQPLqA7IzdSesaDr2oo1SJ1CSoCQfitYAUyu643fyh+h+sjqGU5ZoENdE9kMI6nnCcBlUaWB1DKcs1CdY9kEI7mXiS0EqhVsdQynJ6CFNIWdlZnEs+R0jFEKujKGW5uhXrkpieSHJGskPX67EF5OzVs1QrV41SfqWsjqKU5XzEh1rlaxGbEuvY9Tp0bW7kRMIJ6lWqZ3UMpdxGjcAaxCZrASmQU0mn9PyHUjnUCKzh8JvJPLaAxKfGE1y24F/7V8rT1QysqYcwBZWYnkil0pWsjqGU29A9kELQAqLUb+keSCEkpidSuUxlq2Mo5TYqlKrg8EZTHl1AdA9Eqf8p7VeatMw0h67TVX1hSonIfPvrW0QkNMdrL9qnHxKRPo7IA5CQnkDFUhUdtTqlSrzSfqVJz0p36Dpd1Rfmj0CCMaYB8D7wtn3ZptiGQGwG9AU+ta+v2K5dv0Zpv9KOWJVSHsEtCwgF6Atj//0L+/OFQA8REfv0ecaYDGPMCeCofX1KKQdz1wJSkL4wv85jH0M1CahawGWVUg6QcT2DnRd2OnSdruoLk9c8hekpU+TGUkop2xdMHc1VfWF+nUdE/ICKwJUCLgsUvbGUUsqmfEB5WtVo5dB1uqQvjP330fbnw4DVxtYyfAkw0n6Vph4QDmx1QCal1E3Ss9IdfmGh2KMNG2OyRORGXxhfIPJGXxgg2hizBJgJzBaRo9j2PEbal90nIguwNZPKAp40xlwvbiYAPx8/MrOd085PqZIoLSvN/QoIFKgvTDowPI9lJwITHZEjp0qlK5GYngjZ2eDjsffLKVVgztgD8by/rIMHoWNHKq3YQOJDw+GOO6xOpJRbcMtDGLfj4wNbtlApCBKzgXi9YqMUQMq1FMr5l3PoOj1vDyQoCIDK6ZBYGoiPtzaPUm7iQsoFagTWcOg6Pa+AVKoEvr5USoeE0kByMqQ79u47pUoiLSAF4eMDwcFUSYPLZe3T4uIsjaSUO4hNiaVmYE2HrtPzCghAzZqEJMGpG1/GPZ/rvWlKeRXdAymoOnWolwAnbowndObMLWdXyhvEJsdSs7zugeTvttsISYJz5SHLBzh50upESlnKGMP55PN6CFMg9epR6joEp9qKCMePW51IKUtdSLlAKb9SDh/m0zMLSFgYwP8OY44csTaPUhY7GH+QJkFNHL5ezywgDRsCUD8BjlfGdneqUl7sQPwBGgc1dvh6PbOAhIWBnx/NL8Ke6tiuwiQmWp1KKcscuHRA90AKLCAAGjWi9QXYeeOqVUyMpZGUstKB+AM0CdYCUnAtW9I6FnbVgGwBdjp2KDelSpL9l/brHkihtG5N1TSolG4/D7J9u9WJlLLE2atnyczOJKRiiMPX7bkFpG1bgP8dxmzVgc6Ud9p0ZhMRdSOwNUJwrGIVEBGpIiIrROSI/efvLjKLSCsR2SQi+0Rkj4jcl+O1z0XkhIjssj8cN2BjmzYgQutY2F4L25WYhASHrV6pkmLjmY3cUcc54+IUdw/k78AqY0w4sMr++81SgYeNMTeaR30gIjl7Tj5vjGllf+wqZp7/qVABmjen0xlYf2PPbeNGh61eqZJi49mNRNSNcMq6i1tAcjaM+gIYcvMMxpjDxpgj9ufngYuAa4ZVv/NOIs7A7hqQHACsW+eSzSrlLtIy09h7cS9ta7V1yvqLW0CqG2NiAew/q91qZhFpDwQAx3JMnmg/tHlfREoVM89vde5M2Uxodw42hABr1jh09Uq5u+jz0TQLbkZZ/7L5z1wE+RYQEVkpIntzedzcvjK/9dQEZgOPGGOy7ZNfBBoD7YAqwAu3WL7wjaW6dgWg+wlYVR/blRi9oUx5kR+P/Ui30G5OW3++BcQY09MY0zyXx7dAnL0w3CgQF3Nbh4hUAJYBrxhjNudYd6yxyQA+4xZ9cYvUWKpGDWjWjB4nYHU9bCO0r11bsGWV8gDLjixjQMMBTlt/cQ9hcjaMGg18e/MM9mZT3wCzjDFf3fTajeIj2M6f7C1mnt/r1Yt2VVpwrEYA8d/MgZ49Hb4JpdzR+eTznEo6xR11ndeZoLgF5C2gl4gcAXrZf0dE2orIDPs8I4DOwJhcLtfOEZEYIAYIAv5VzDy/9+67+O/aQ8+mA1gWmgmBgQ7fhFLuaPmR5fQO642fj/OaLxRrzcaYy0CPXKZHA+Psz78Evsxj+e7F2X6B2JtKDWsyjDkxcxjdanQ+CyjlGZYeXsqwpsOcug3PvRP1Jnc3vJv1p9eTlJ5kdRSlnC49K501J9fQt0Ffp27HawpIhVIV6Brale8Of2d1FKWcbtnhZbSp2YagskFO3Y7XFBCwHcYs3L/Q6hhKOd3sPbN5qOVDTt+OVxWQgY0GsvrEaq5mXLU6ilJOE58az5qTa7i36b1O35ZXFZBKpSvRK6wXc2PmWh1FKaeZv3c+d4ffTYVSFZy+La8qIADjbx/PtB3TrI6hlNO46vAFvLCA9ArrxZW0K0Sfj7Y6ilIOdyj+ECcTT9IrrJdLtud1BcRHfHj09keZtl33QpTn+Xjrx4xtPdapN4/l5HUFBGBs67F8tf8rPZmqPEpieiJzYubwZLsnXbZNrywgNQJr0L1ed77ck+sNskqVSJE7I+kX3o/aFWq7bJteWUAAnu34LJM2TiIrO8vqKEoVW1Z2Fh9u+ZCnOzzt0u16bQHpFNKJuhXrMn/vfKujKFVsSw4toXaF2rSvneeIGE7htQUE4KU7X+LNDW+S/ev4RkqVPMYYJm2cxF87/NXl2/bqAtI7rDel/Eqx9PBSq6MoVWQ/HP2BpIwk7mlyj8u37dUFRER46c6XmLh+IsYYq+MoVWjGGF5Z8wqvd30dXx9fl2/fqwsIwNAmQ0nOSOb7o99bHUWpQlt8cDHGGIY2GWrJ9p3eWMo+3/Uco5EtyTG9nohssS8/3z78oUv5iA9v93yb51c8r1dkVIlyPfs6r655lTe6vYGPWLMv4IrGUgBpOZpHDcox/W3gffvyCcAfi5mnSAY0HEC1ctWI3BlpxeaVKpIF+xZQvlR5+of3tyyD0xtL5cU+kHJ34MYAHYVa3pFEhEm9JvHa2tdIzki2IoJShZKamcqLq17kzR5vOqXnbUG5qrFUaXtPl80icqNIVAUSjTE3jhvOAq67he4mbWq1oVf9Xrzz8ztWRVCqwN5c/yYd63Ska2hXS3Pk+40bEVkJ1MjlpZcLsZ0QY8x5EakPrLaPxJ7bF1HyvBQiIuOB8QAhISF5zVYs/+r+L1pPbc2jbR4lpKJztqFUcR25fITJ0ZPZ/afdVkdxTWMpe09cjDHHgbVAayAeqCQiN4pYHeD8LXIUvrFUIYVUDOGvHf7K48se18u6yi0ZY3j6h6d5odMLLv3OS15c0Viq8o2etyISBHQC9hvbX+gaYNitlne1F+58gdNJp5m3d57VUZT6nW8PfcuJxBM83dG133nJiysaSzUBokVkN7aC8ZYxZr/9tReAZ0XkKLZzIjOLmafYAnwDmDFwBs9EPUN8arzVcZT6VVJ6Ek//8DQf9/uYAF+X3/GQKymJu+pt27Y10dHOHVHsmR+e4XLaZWYNneXU7ShVUI98+wgBPgFMHTjV6dsSke3GmLb5zef1d6Lm5Y3ub7D+9Hp+OPqD1VGUYvHBxaw7tY53+7xrdZTf0AKSh8CAQKYPnM64JeO4+Euu54aVcomLv1zk8WWPM2vILAID3Ku3sxaQW+hZvycPtnyQMYvH6Ff+lSWMMYz/bjyj/zCaTiGdrI7zO1pA8vFGtzdISE/g/U3vWx1FeaEZO2ZwIvEE/+z6T6uj5EoLSD78ff2Ze+9c3v75bbae22p1HOVFos9H8/Lql5k/bD6l/EpZHSdXWkAKILRSKFMGTGHkwpEkpSdZHUd5gfjUeIYtGMaUAVNoHNTY6jh50gJSQPc0uYcBDQcwatEo/dq/cqqs7CxGLhzJyOYjLRllrDC0gBTCu73fJTM7k+d+fM7qKMqDvbL6FcD23Sx3pwWkEPx9/flq+Ff8cPQHpkY7/2Ye5X3mxsxl7t65zL13rsu6yxWH+yd0M5VKV2Lp/UvpFNmJBlUa0KN+D6sjKQ+x6vgqnv7haVY9vIrgcs75wqij6R5IETSo0oD5w+Zz/9f3czD+oNVxlAfYdWEXoxaNYsHwBbSo3sLqOAWmBaSIuoZ25Z2e79B7dm9OJJywOo4qwU4mnmTAfwfwSf9PLB8gqLD0EKYYRrcaTcq1FHrO7sm6MevcYnwGVbJcTr1M3y/78rdOf2N4s+FWxyk0LSDF9GT7J/kl8xd6zu7JT2N+olq5vEZ1VOq3LqdepufsngxtPJS/dPiL1XGKRA9hHOBvnf7G8KbD6T27NwlpCVbHUSXApV8u0X1Wd3rX782/e/zb6jhFpgXEQf7Z9Z90r9edXrN7cemXS1bHUW4sLiWObl90Y2DDgbzV8y1LR1UvLqc3lhKRbjmaSu0SkfQbI7OLyOciciLHa62Kk8dKIsK7vd+lT1gfOn/emTNJZ6yOpNzQhZQLdPuiG8OaDuONbm+U6OIBLmgsZYxZc6OpFLY+MKnAjzlmeT5H06ldxcxjKRFhYo+JPHr7o9z52Z16iVf9xomEE3T5vAujmo9iQtcJJb54gOsbSw0DvjfGpBZzu25ZkKw2AAALmUlEQVTt2Tue5Z9d/0nXz7sSfd65Qy+qkmHrua10iuzEn9v/mVe7vGp1HIdxVWOpG0YCc2+aNlFE9ojI+zdGb8+NiIy3N6eKvnTJ/c8xjGk1hqkDptJvTj8dFtHLfXPgGwb8dwDTBk7jqfZPWR3HsYwxt3wAK4G9uTwGY+ssl3PehFuspyZwCfC/aZoApbDtwfwjvzzGGNq0aWNKig2nNpiak2qadza8Y7Kzs62Oo1woOzvbvLfxPVP73dom+ly01XEKBYg2BfhbzPc+EGNMz7xeE5E4EalpjIm9VWMpuxHAN8aYzBzrjrU/zRCRzwCP+5prp5BObBm3hSHzh7ArbhczBs6gjH8Zq2MpJ8vIyuCZqGdYd2odG/+40WM7HTq9sVQOo7jp8CVHVzvBdv5kbzHzuKW6Feuy4ZENANz12V16hcbDnUw8yV2f3cWFlAv8PPZnjy0e4JrGUohIKFAX+Omm5efY++TGAEGA+w+AUERl/Mvw5dAvua/ZfXSY0YEVx1ZYHUk5wdLDS+kwowMjm49k0YhFVCxd0epITqWNpSyw+sRqRi8ezYimI/h3j3+77XiXquCysrN4edXLzN07l3nD5hFRN8LqSMWijaXcWPd63dn12C5OJJ6gw4wO7L+0P/+FlNs6fPkwXT7vwu643ex4bEeJLx6FoQXEIlXLVmXRiEU81f4punzehU+3fUpJ3Bv0Ztezr/PepveImBnByGYjWf7AcoLKBlkdy6X0EMYNHL58mAe+foDAgEAm3z3ZrUfhVjaH4g8xdslY/Hz8iBwUSViVMKsjOZQewpQgDas2ZNMfNzGk0RDujLyTf6z5B+lZ6VbHUrnIvJ7JpI2T6BTZiZHNRrJm9BqPKx6FoQXETfj5+PF0x6fZ/afd7L+0nxaTW7Dy+EqrY6kcfjj6Ay2ntGTF8RVsGbeFP3f4Mz7i3X9COqCQm6ldoTYLRyxk2eFlPPrdo7Sr1Y6J3ScSXjXc6mhe6/Dlwzwb9SyHLh/i/T7vc3f43R7xRThH8O7y6cbubng3+57YR6sarbhj5h08sewJLqRcsDqWV0lIS+C5H58jYmYEXUO7su+JfQxoOECLRw5aQNxYWf+yvHTXSxx66hBl/cvS7NNmvLL6FW2v6WSXUy/zyupXaPBRA65mXGXfE/t4LuI5AnwDrI7mdrSAlABVy1ZlUu9J7HxsJ+eSzxH+UTgT1k7Qkc8cLD41npdWvUTDjxsSlxJH9KPRTBs4jeqB1a2O5ra0gJQgIRVD+GzwZ6x7ZB3nrp6j4ccNeWLZExy9ctTqaCXa6aTTvLDiBRp93IgraVfYPn470wdNp17lelZHc3taQEqgxkGNmT5oOgeePECVMlW4Y+YdDFswjE1nNunNaAVkjGHl8ZUMnT+U1lNbk56Vzs7HdjJlwBRCK4VaHa/E0BvJPEDKtRQid0by0daPCPANYGyrsTzY8kHd9c5FUnoSs3bP4tPoT/H38efJdk/yQEvbTXzqfwp6I5kWEA9ijGHD6Q1E7orkmwPf0K1eN8a2Gku/8H4lolGzs6RlprH8yHLm7ZvHj8d+pG+DvjzV7inuDLlTr6jkQQuIl0vOSGbBvgVE7ork8OXDDGg4gMGNBtM7rDdl/ctaHc/pMq9nsvL4Subunct3h7/j9pq3M6r5KO5pcg9VylSxOp7b0wKifnUq8RRLDi1h8aHFbDu3jW71ujGk0RD6h/f3qMOck4kniToaRdSxKNacXEPjoMaMaj6K4U2HU7N8TavjlSguKSAiMhyYADQB2htjcv2rFpG+wH8AX2CGMebGwEP1gHlAFWAH8JAx5lp+29UCUnRX0q6w/MhyFh9czMrjK6lZviadQzrTJbQLnW/rTJ0KdayOWGCxybFsPbeVVSdWEXUsioS0BHqH9aZPWB96hfWiRmANqyOWWK4qIE2AbGAq8FxuBUREfIHD2EYsOwtsA0YZY/aLyALga2PMPBGZAuw2xkzOb7taQBzjevZ19sTtYd2pdaw7vY51p9ZRPqA8EXUjaFm9JS2qtaBF9RbULl/b8nMFiemJbD+/nW3nt7H13Fa2nd9GamYq7Wq1o8ttXejToA+tarTy+u+mOIpLD2FEZC15F5A7gAnGmD7231+0v/QWtlHaaxhjsm6e71a0gDiHMYYD8QfYfHYzMXEx7Lm4h5i4GLKys2herTnNqzXntoq3UadCHepWrEvdCnWpXaG2Q+7QzMjKID41nrhf4jh65ShHLh/hyBX74/IRUjNTaV2zNe1qtaN97fa0q9WO+pXrW17YPFVBC4grTs3XBnKOInwW6ABUxdYWIivH9NouyKPyICI0DW5K0+Cmv5kelxJHzMUY9l/az5mkM+y8sJOzV89y5uoZYpNjqVymMpVLVyYwIJDAgEDKlypve+4fiK+PL1nZWb8+MrMzycrOIi0zjfjUeC6lXiI+NZ6MrAyCywUTXDaYsCphhFcJ566QuxjbeizhVcKpEVhDi4UbyreAiMhKILeDyZeNMbcahf3XVeQyzdxiel45xgPjAUJCPHeUa3dUPbA61QOr07P+7zt8XM++zsVfLpKUkUTKtRSSM5JJuZZie34tmWyTjZ+P3+8eZfzKEFQ2iKCyQQSXC6Z8QHktECVQsfrCFNBZbCOy31AHOA/EA5VExM++F3Jjel45pgHTwHYIU8xMykF8fXypWb6mXuXwUq4447QNCBeReiISgK295RJ796s12PrlQv59ZZRSbqZYBUREhorIWeAOYJmIRNmn1xKR5QD2vYungCjgALDAGLPPvooXgGdF5Ci2cyIzi5NHKeVaeiOZUup3dFBlpZTTaQFRShWZFhClVJFpAVFKFZkWEKVUkZXIqzAicgk4VYBZg7DdsFaSecJ7AM94H970Hm4zxgTnN1OJLCAFJSLRBbkU5c484T2AZ7wPfQ+/p4cwSqki0wKilCoyTy8g06wO4ACe8B7AM96HvoebePQ5EKWUc3n6HohSyok8qoCIyHAR2Sci2SKS55lmEekrIodE5KiI/N2VGfMjIlVEZIWIHLH/rJzHfNdFZJf9scTVOXOT3+cqIqVEZL799S0iEur6lPkrwPsYIyKXcnz+46zIeSsiEikiF0Vkbx6vi4h8aH+Pe0Tk9iJtyBjjMQ9so8M3AtYCbfOYxxc4BtQHAoDdQFOrs+fI9w7wd/vzvwNv5zFfitVZC/u5Ak8AU+zPRwLzrc5dxPcxBvjY6qz5vI/OwO3A3jxe7w98j21kwI7AlqJsx6P2QIwxB4wxh/KZrT1w1Bhz3NhaSMwDBjs/XYENBr6wP/8CGGJhlsIoyOea870tBHqI+41j6O7/fxSIMWYdcOUWswwGZhmbzdhGByz0sHIeVUAKKLdBnt1pMOfqxphYAPvPannMV1pEokVks4i4Q5EpyOf66zzGNtBUEraBpNxJQf//uNe+679QROrm8rq7c8jfQYlrmOrEQZ5d5lbvoRCrCTHGnBeR+sBqEYkxxhxzTMIiKcjnavlnXwAFyfgdMNcYkyEif8K2V9Xd6ckcyyH/LUpcATHOG+TZZW71HkQkTkRqGmNi7buUF/NYx3n7z+P2vjytsR27W6Ugn+uNec6KiB9QkVvvZlsh3/dhjLmc49fpwNsuyOVoDvk78MZDmFwHebY4U05LsA0wDXkMNC0ilUWklP15ENAJ2O+yhLkryOea870NA1Yb+xk9N5Lv+7jpXMEgbGP9ljRLgIftV2M6Akk3Dp0LxeqzxQ4+8zwUW2XNAOKAKPv0WsDym85AH8b2L/bLVue+6T1UBVYBR+w/q9int8XWVxggAojBdoUgBvij1bnz+lyB14FB9uelga+Ao8BWoL7VmYv4Pt4E9tk//zVAY6sz5/Ie5gKxQKb9b+KPwJ+AP9lfF+AT+3uMIY+rlvk99E5UpVSReeMhjFLKQbSAKKWKTAuIUqrItIAopYpMC4hSqsi0gCilikwLiFKqyLSAKKWK7P8DKSgJY4WSIcgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7f56644e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARAAAAD8CAYAAAC/+/tYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4lGXa9/HvGVIghBJI6MRQAgSBDVINSu9SNSLYQBZwraivZW2rq8taHizrWqhRQR+KoIiARiAgIDVIB6mhhxIIITEhJOR6/5jBJ2JC2szck5nzcxxzZHLPXX4zmpO7zXWKMQallCoJH6sDKKXKLi0gSqkS0wKilCoxLSBKqRLTAqKUKjEtIEqpEtMCopQqMS0gSqkS0wKilCoxX6sDlERISIgJDw+3OoZSHmvz5s3JxpjQwuYrkwUkPDychIQEq2Mo5bFE5EhR5tNDGKVUiWkBUUqVmBYQpVSJaQFRSpWYFhClVIk5pICISKyInBGRnQW8LiLygYgcEJHtInJTntdGish++2OkI/IopVzDUXsgnwF9r/N6PyDC/hgHfAIgItWAV4AOQHvgFREJdlAmpZSTOeQ+EGPMKhEJv84sg4EZxjZ+4noRqSoitYGuwFJjzHkAEVmKrRDNckQu5Tq5JpeM7AzSstJIv5z+++OKuYKvj++fHhV8KxASGELlgMqIiNXxVQm56kayusCxPL8ft08raPqfiMg4bHsvhIWFOSelypcxhhNpJ9h9djfHUo9x7OIxjl88/vvP4xePk345nQq+FQjyD6JSQCWC/IOo6FcRXx9fcnJzfn9k52aTk5tDRnYG5zLOcSnnEiGBIYQEhhBaMZQaFWvQsGpDIqpHEFEtgojqEYQGhmqRcVOuKiD5/dc315n+54nGTAGmALRt21ZHgnaSzOxMtpzawo7TO9h+ejs7zuxg55md+Pr4cmONGwmvGk69SvVoX7c9d0TeQb3K9ahXuR5VylfBR4p/RHwp5xLnMs6RnJFMckYyp9JPcTDlIMsTlzMpYRL7z+8nJzeHiGoRtK7VmnZ129G+bntuDL0Rv3J+TvgEVHG4qoAcB+rn+b0ecNI+ves101e6KJMC0rLSWHtsLauOrGLV0VVsSdpCs5BmtKrZipY1WjI0cigta7SkZlBNp2y/vG956lauS93K+e54AnA+8zx7k/fyS9IvrD22lvfXv8/R1KP8pdZfaFenHV1u6EKPhj2oHFDZKRlVwcRRbR3s50AWGWNa5PPabcCjQH9sJ0w/MMa0t59E3QxcvSrzC9Dm6jmRgrRt29bod2FKxhjD5qTNLPh1AXEH49hzdg9t6rShc1hnuoR3oWO9jgT5B1kds1AXsy6y+eRmNpzYQHxiPOuOryOqVhR9GvWhT6M+3FT7Jsr5lLM6ZpklIpuNMW0Lnc8RBUREZmHbkwgBTmO7suIHYIyZJLYD2A+xnSDNAB4wxiTYlx0NvGBf1QRjzKeFbU8LSPFcvnKZnw7/xIJfF7Bw30IC/QIZ0nQI/SP606FeB8r7lrc6YqllZGew6sgq4g7EEXcwjjO/nWFg04GMaDGC7g264+tTJr83ahmXFhBX0wJSOGMMa4+tJXZLLN/8+g1NQ5oyuOlghjQbQrOQZlbHc7qjqUeZv3s+s3fN5vCFw8RExjCi5Qii60eX6FyNt9EC4qWS0pKYsW0GsVtjEYTRrUdzb6t7qVOpjtXRLHPw/EFm75zN7F2zSb2UygNRDzCuzbjrnnfxdlpAvIgxhqWHlvLfjf9lzdE1xETGMLr1aDrW66iXP6+x/fR2JidMZtbOWfRo2INH2j1Clxu66Od0DS0gXiD7Sjazd85m4rqJ5Jpcnuz4JHfdeBcV/StaHc3tXcy6yMxtM/lo00f4iA+PtHuEUVGjqOBXwepobkELiAe7mHWRqZun8v6G94moFsEz0c/Qt3Ff/Ve0BIwxrDi8gv9s+A+bTmzimehneLDtgwT6BVodzVJFLSB6NqkMyczO5K01b9Hog0ZsOrmJBXctIH5kPP0i+mnxKCERoXuD7nw7/FuW3LOE1UdX0+iDRryz9h1+u/yb1fHcnhaQMiAnN4epm6cS8d8INp3cxOoHVjM7ZjZt6rSxOppHiaoVxdd3fU3cvXGsO76ORh804t1173L5ymWro7ktLSBuzBjD13u+psXHLfjfnf/L/GHzmTdsnldchrVSq5qtmDdsHkvvW0p8YjwtPm7Bon2LKIuH+86m50Dc1J6ze3ho8UNcuHSBN3u+SZ9GffQwxSLf7/+eJ+OeJLxqOO/1eY/I0EirIzmdngMpozKzM3kp/iU6f9aZmOYxbB63WU+QWqxfRD92PLSDvo370vmzzoz/fjwXsy5aHcstaAFxI3EH4mjxSQv2ndvHtr9t49H2j+r3OdyEXzk/nuj4BLsf3k365XRaftKSpQeXWh3LcvoFATeQkpnCo98/yrpj6/io/0f0i+hndSRVgNCKoUwfPJ24A3GM+W4MfRr1YWLviV77TWDdA7HYT4d/ImpyFNUrVGfnwzu1eJQRfRr3YfvftgPQ8pOW/HjwR4sTWUP3QCySfSWbV1e+SuzWWKYPmk7/iP5WR1LFVKV8FaYMnMKPB39k7HdjGdx0MBN7T8S/nL/V0VxG90AssP/cfjrFdmLr6a1sfXCrFo8yrnej3mx9cCtHU49y66e3cuRCkdrKegQtIC727a/f0im2E/f/5X4WjVjktJG+lGsFVwjmm7u+YVjzYbSf1p7F+xZbHcklHHIIIyJ9gf8A5YBpxpg3r3n9PaCb/ddAoIYxpqr9tSvADvtrR40xgxyRyd3kmlwmrJrAlF+msOjuRbSv297qSMrBRIT/F/3/6FivI8PnD+feo/fyevfXPXowo1LfSCYi5YB9QC9sY5xuAkYYY3YXMP9jQGtjzGj77+nGmGKNoVfWbiRLv5zOqAWjOJF2gq+HfU3tSrWtjqSc7OxvZ7n767vx9fFlbsxcKgVUsjpSsbjyRrL2wAFjzCFjzGVgNrY+MAUZgRf1fUlMSSR6ejSVAyqzcuRKLR5eIrRiKN/f8z1hlcO49dNbOX7xuNWRnMIRBaQ4vV1uABoA8XkmlxeRBBFZLyJDHJDHbfyS9AvRsdGMuWkM0wdNJ8A3wOpIyoV8fXyZNGASd7e8m+jp0Ww7tc3qSA7niIOzIvd2AYYD84wxV/JMCzPGnBSRhkC8iOwwxhz800bKWGOpnw7/xJ1f3cmkAZO4PfJ2q+Moi4gIz3Z6lvCq4fSa2YsZQ2fQt/H1usCWLY7YAymo50t+hnPN4Ysx5qT95yFsPWFa57egMWaKMaatMaZtaGhoaTM71cK9C4n5KoZZd8zS4qEAGHbjMBYMX8CoBaOYs3OO1XEcxhEFZBMQISINRMQfW5FYeO1MItIUCAbW5ZkWLCIB9uchQCcg35OvZcWMbTMY9904lty9hB4Ne1gdR7mR6PrRLL1vKU/GPcmX27+0Oo5DlPoQxhiTIyKPAnHYLuPGGmN2ichrQIIx5moxGQHMNn+87BMJTBaRXGzF7M2Crt6UBZMTJjNh9QRWjFzhFV/5VsXXsmZLlt2/jF4ze5GTm8PIqJFWRyoVHQ/EQWZsm8GL8S+ycuRKGlVrZHUc5eZ+Tf6VnjN68lq31xjderTVcf6kqJdxPfcOFxeav3s+zy17jvj747V4qCJpFtKM+JHx9JhhO8x1xyJSFFpASmnJ/iU8vORh4u6N08MWVSxNqjdh+f3L6fJZF0IDQxnYdKDVkYpNvwtTCisPr2TUglF8O/xbompFWR1HlUFNqjfh2+HfMnrhaNYfX291nGLTAlJCe87uYdhXw5gTM4eO9TpaHUeVYe3rtufzIZ8zZPYQ9ibvtTpOsWgBKYFzGecYOGsgb/d6m24NuhW+gFKF6B/Rnzd7vknfL/uSlJZkdZwi0wJSTJevXOb2ubdzR+QdjIoaZXUc5UFGRY1iTOsxDJg1gMzsTKvjFIkWkGIwxvDQoocILh/MGz3fsDqO8kAv3PoCEdUieHjJw2WiD40WkGJ4f/37bE7azBe3f4GP6EenHE9EmDZoGptObGLy5slWxymUXsYtonXH1vHmz2+yccxGgvyLNXyJUsUS5B/EN3d9Q6fYTkTVinLrk/T6z2gRpGSmMGL+CKYOnMoNVW+wOo7yAhHVI5g+aDp3fnUnp9NPWx2nQFpACmGMYcx3YxjcdDCDmnrkaIvKTQ1sOpCRfxnJfd/cR67JtTpOvrSAFOKThE9ITEnk7V5vWx1FeaFXu75KalYqn2z6xOoo+dICch3bT2/nlZWvMCdmjo4mpizh6+PLzKEzefWnV93yJjMtIAXIyc3hgW8f4K2ebxFRPcLqOMqLNanehFe7vMr9C+4nJzfH6jh/oAWkAO+te49qFarxQNQDVkdRiofaPUSVgCq8sewVeOwxmDfP6kiAXsbN14HzB3jr57fYOHYjIvkN+aqUa/kgxJpB3BQ/nsFf5dLq66+hd2+obG1Tb4fsgYhIXxHZKyIHROTv+bw+SkTOishW+2NMntdGish++8Py4ZmMMYz7bhzP3/I8DYMbWh1HKZvDh6k35ileW57LI/3BnDwJL79sdarSFxB7Y6mPgH5Ac2CEiDTPZ9Y5xpgo+2OafdlqwCtAB2z9ZV4RkeDSZiqN2C2xpF1OY3zH8VbGUOqPGjSAZ55h7GbI8IMvWgEffgjbrG0VYUVjqbz6AEuNMeeNMSnAUsCyMe9TMlN4fvnzTB041aPbEaoy6sUXKXdDOB8vhud6QapfLjz+OFj4nRlXNpa6Q0S2i8g8EbnaBqLITalcYcLqCQxpNkQHB1LuKTAQPviADieg/354tSuwahV8/bVlkRxRQIrSWOo7INwY0wpYBnxejGVtM4qMs3ewSzh79myJwxbkUMohPt36Ka91e83h61bKYQYMgL59eWMZfNkKdtQAnn0WLl+2JI5LGksZY84ZY7Lsv04F2hR12TzrcGpjqReWv8ATHZ6gVlAth69bKYcRgXffJTSrHC+ughd7AIcOwWRrvrnrksZSIpK3o/QgYI/9eRzQ295gKhjobZ/mUuuPr2fN0TU8dfNTrt60UsUXGQnjxvHgZthSCzbUBf71L0hPd3mUUhcQY0wOcLWx1B5g7tXGUiJy9dtnj4vILhHZBjwOjLIvex54HVsR2gS8Zp/mMsYYnln6DP/q/i8q+ld05aaVKrl//IPyARV5eRW83B04c8Z2VcbFvL6x1MrDKxn33Tj2PLKHcj7lHLJOpVzixRfJfvPfNHsUYr+FLunV4fBhCCr9eDVFbSzl9bey/3v1v/n7LX/X4qHKnqefxi+oMq/8ZNsLMefOwaRJLo3g1QVk04lN/Jr8K/e2utfqKEoVX3AwPP4492yHMxVhRQPg3XchK6vQRR3FqwvIG2ve4Onop/Ev5291FKVK5oknKFchkKfXwnsdgaQk+PJLl23eawvI7rO7+fnYz4y5aUzhMyvlrqpXh7FjuWc7bKgH+6th2wtx0blNry0g7657l8faP0agX6DVUZQqnSeeoIIpx7jN8EEHYNcuiI93yaa9soCkXkpl3u55jL1prNVRlCq98HC4/XYe3mS7O/VCeVx2SdcrC8iXO76kV6Ne1AyqaXUUpRzjsceokwb99sP01sDChXD8uNM363UFxBjD5M2TebDNg1ZHUcpxbrkFWrbk8Q3wSTswubnw2WdO36zXFZCNJzaSfjmd7g26Wx1FKccRgXHjaH8C/K7AuvrA9OmQ69x2EF5XQKZsnsK4m8Zpa0rlee65BwkI4L7tMLMVcPSo0wcc8qq/oszsTObvmc/IKMtHTlTK8YKDISaGey7UZ267CmQl7ofWrZ26Sa8qIHEH42hTp41+ZV95ro8+4oath2kV3oHFv211+ua8qoDM2z2PmMgYq2Mo5TxVqoCPD/e1uo+Z22c6fXNeU0CycrJYvH8xQyOHWh1FKae7I/IOlh1aRvpl544R4jUFZOmhpbSq2UoPX5RXqFK+Ch3qdmDZoWVO3Y7XFBA9fFHeZkCTASzet9ip23BVY6mnRGS3fVT25SJyQ57XruRpOLXw2mUdIdfksnj/YoY0G+KM1Svllm6LuI0lB5bgzEHDXNVYagvQ1j4q+zzg7TyvZeZpODUIJ9h+ejvVKlSjfpX6hc+slIeIqB5BRb+KbD3lvKsxLmksZYxZYYzJsP+6Htvo6y4TnxhPjwY9XLlJpdzCbRG3sWjfIqet35WNpa76K/B9nt/L2/u9rBeRAo8xStMXZnnicr11XXmlvo37svTQUqet31WNpWwzitwLtAX+J8/kMPvgrXcD74tIo/yWLWlfmOwr2aw5uoau4V2LvIxSnqJjvY78kvQL2VeynbJ+lzSWAhCRnsCLwKA8TaYwxpy0/zwErAQceu/tppObaBjckJDAEEeuVqkyoUr5KjQMbui08yCuaizVGpiMrXicyTM9WEQC7M9DgE7Abgdk+t3qI6vpHNbZkatUqkyJrh/NuuPrnLJuVzWW+h8gCPjqmsu1kUCCveHUCuBNY4xDC8iWU1toW6fQ9hZKeazo+tGsPbbWKev2dcRKjDFLgCXXTPtHnuc9C1huLdDSERkK8kvSL7zU+SVnbkIpt3ZzvZt5Kd45fwMefSdqWlYaJ9JO0CykmdVRlLJM42qN+S37N06ln3L4uj26gGw7vY0WNVrg6+OQHS2lyiQRoXloc/ac3VP4zMXk0QVkS9IWWtdy7oAqSpUFkSGR7EnWAlIs205v4y81/2J1DKUsFxkSqXsgxXUo5RCNqzW2OoZSlosM1T2QYjt84TDhVcOtjqGU5fQQpphycnM4kXaCsCphVkdRynL1q9TnwqULpGWlOXS9HltAjl88To2KNQjwDbA6ilKW8xEf6lSqQ1J6kmPX69C1uZHElEQaVG1gdQyl3EatoFokpWkBKZIjqUf0/IdSedQKquXwm8k8toAkZyQTGlj0r/0r5elqB9XWQ5iiunDpAlXLV7U6hlJuQ/dAikELiFJ/pHsgxXDh0gWCKwRbHUMpt1E5oLLDG015dAHRPRCl/k953/JkZmc6dJ2u6gsTICJz7K9vEJHwPK89b5++V0T6OCIPQMqlFKoEVHHU6pQq88r7ludSziWHrtNVfWH+CqQYYxoD7wFv2Zdtjm0IxBuBvsDH9vWV2uUrlynvW94Rq1LKI7hlAaEIfWHsv39ufz4P6CEiYp8+2xiTZYxJBA7Y16eUcjB3LSBF6Qvz+zz2MVRTgepFXFYp5QBZV7LYcmqLQ9fpqr4wBc1TnJ4yJW4spZSyfcHU0VzVF+b3eUTEF6gCnC/iskDJG0sppWwq+VciqlaUQ9fpkr4w9t9H2p/HAPHG1jJ8ITDcfpWmARABbHRAJqXUNS7lXHL4hYVSjzZsjMkRkat9YcoBsVf7wgAJxpiFwHRgpogcwLbnMdy+7C4RmYutmVQO8Igx5kppMwH4+viSneucdn5KlUWZOZnuV0CgSH1hLgF3FrDsBGCCI3LkVbV8VS5cuuDo1SpVZjljD8Rj70TVAqLUH2kBKYaqAVpAlMor/XI6Ff0qOnSdHltAgisEawFRKo9T6aeoFVTLoev02AJStXxVUjJTrI6hlNvQAlIM1SpU41zmOatjKOU2ktKTqB1U26Hr9NgCElYljCOpR6yOoZTb0D2QYmhQtQGJKYlWx1DKbSSlJVG7ku6BFElYlTBOpJ1wyv3/SpU1xhhOpp3UQ5iiCvANIDQwlBMXT1gdRSnLnUo/RYBvgMOH+fTYAgLQILgBiRf0MEapX5N/JTIk0uHr9egC0jC4IYdSDlkdQynL7UneQ7OQZg5fr0cXkBahLdh+ervVMZSy3J6ze3QPpLha127t8BGYlCqL9iTvITJUC0ixtK7Vmq2ntpJrcq2OopSldp/drXsgxVU9sDpVy1fV8yDKqx2/eJzs3GzCqoQ5fN0eXUDAtheyJUkPY5T3WndsHdH1o7E1QnCsUhUQEakmIktFZL/9558uMotIlIisE5FdIrJdRO7K89pnIpIoIlvtD8cO2IitgGxO2uzo1SpVZqw9tpab693slHWXdg/k78ByY0wEsNz++7UygPuNMVebR70vInl7Tj5jjImyP7aWMs+fdArrxOqjqx29WqXKjLXH1xJdP9op6y5tAcnbMOpzYMi1Mxhj9hlj9tufnwTOAC4bVj26fjTbTm0jLSvNVZtUym1kZmey88xO2tZp65T1l7aA1DTGJAHYf9a43swi0h7wBw7mmTzBfmjznogElDLPnwT6BdKubjvWHF3j6FUr5fYSTiZwY+iNBPoFOmX9hRYQEVkmIjvzeVzbvrKw9dQGZgIPGPP7ddXngWZAO6Aa8Nx1li9xY6nu4d1Znri8WMso5Ql+PPgj3cK7OW39hRYQY0xPY0yLfB7fAqftheFqgTiT3zpEpDKwGHjJGLM+z7qTjE0W8CnX6YtbmsZSPRr2ID4xvljLKOUJFu9fzIAmA5y2/tIewuRtGDUS+PbaGezNpr4BZhhjvrrmtavFR7CdP9lZyjz5alenHQdTDpKckeyM1Svllk6mneRI6hFuru+cKzBQ+gLyJtBLRPYDvey/IyJtRWSafZ5hQGdgVD6Xa78UkR3ADiAE+Fcp8+TLr5wfPRv2ZPG+xc5YvVJuacn+JfRu1BtfH4e0f8pXqdZsjDkH9MhnegIwxv78C+CLApbvXprtF0dMZAxf7viSkVEjC59ZKQ+waN8iYprHOHUbHn8n6lW3NbmN1UdXk3op1eooSjndpZxLrDi8gr6N+zp1O15TQCoHVKZreFe+2/ed1VGUcrrF+xbTpnYbQgJDnLodrykgYDuMmbd7ntUxlHK6mdtncl+r+5y+Ha8qIAObDiQ+MZ6LWRetjqKU0yRnJLPi8AruaH6H07flVQWkavmq9GrUi1k7ZlkdRSmnmbNzDrdF3EblgMpO35ZXFRCAcTeNY8ovU6yOoZTTuOrwBbywgPRq1IvzmedJOJlgdRSlHG5v8l4OXzhMr0a9XLI9rysgPuLD2JvGMmWz7oUoz/Phxg8Z3Xq0U28ey8vrCgjA6Naj+Wr3V3oyVXmUC5cu8OWOL3mk3SMu26ZXFpBaQbXo3qA7X2zP9wZZpcqk2C2x9IvoR93KdV22Ta8sIABPdXyKiWsnau9c5RFycnP4YMMHjO8w3qXb9doC0imsE/Wr1GfOzjlWR1Gq1BbuXUjdynVpX7fAETGcwmsLCMALt7zAG2ve0L4xqkwzxjBx7USe6PCEy7ft1QWkd6PeBPgGsGjfIqujKFViPxz4gdSsVG6PvN3l2/bqAiIivHDLC0xYPQFjjNVxlCo2YwwvrXiJ17q+Rjmfci7fvlcXEIChkUNJy0rj+wPfWx1FqWJb8OsCjDEMjRxqyfad3ljKPt+VPKORLcwzvYGIbLAvP8c+/KFL+YgPb/V8i2eWPqNXZFSZciX3Ci+veJnXu72Oj1izL+CKxlIAmXmaRw3KM/0t4D378inAX0uZp0QGNBlAjYo1iN0Sa8XmlSqRubvmUimgEv0j+luWwemNpQpiH0i5O3B1gI5iLe9IIsLEXhN5ZeUr2oBKlQkZ2Rk8v/x53ujxhlN63haVqxpLlbf3dFkvIleLRHXggjHm6nHDccB1t9Bdo02dNvRq2Iu3f37bqghKFdkbq9+gY72OdA3vammOQr9xIyLLgFr5vPRiMbYTZow5KSINgXj7SOz5fRGlwEshIjIOGAcQFhZWjE0X3b+6/4vWk1szts1Ywqo4ZxtKldb+c/v5JOETtv1tm9VRXNNYyt4TF2PMIWAl0BpIBqqKyNUiVg84eZ0cJW4sVVRhVcJ4osMTPLT4Ib2sq9ySMYbxP4znuU7PufQ7LwVxRWOp4Ks9b0UkBOgE7Da2v9AVQMz1lne15255jqOpR5m9c7bVUZT6k2/3fkvihUTGd3Ttd14K4orGUpFAgohsw1Yw3jTG7La/9hzwlIgcwHZOZHop85Safzl/pg2cxpNxT2onO+VWUi+lMv6H8XzY70P8y7n8jod8SVncVW/btq1JSHDuiGJP/vAk5zLPMWPoDKduR6mieuDbB/D38WfywMlO35aIbDbGtC1sPq+/E7Ugr3d/ndVHV/PDgR+sjqIUC35dwKojq3inzztWR/kDLSAFCPIPYurAqYxZOIYzv+V7blgplzjz2xkeWvwQM4bMIMg/yOo4f6AF5Dp6NuzJva3uZdSCUfqVf2UJYwzjvhvHyL+MpFNYJ6vj/IkWkEK83u11Ui6l8N6696yOorzQtF+mkXghkX92/afVUfKlBaQQfuX8mHXHLN76+S02nthodRzlRRJOJvBi/IvMiZlDgG+A1XHypQWkCMKrhjNpwCSGzxtO6qVUq+MoL5CckUzM3BgmDZhEs5BmVscpkBaQIro98nYGNBnAiPkj9Gv/yqlycnMYPm84w1sMt2SUseLQAlIM7/R+h+zcbJ7+8WmroygP9lL8S4Dtu1nuTgtIMfiV8+OrO7/ihwM/MDnB+TfzKO8za8csZu2cxaw7Zrmsu1xpuH9CN1O1fFUW3b2ITrGdaFytMT0a9rA6kvIQyw8tZ/wP41l+/3JCKzrnC6OOpnsgJdC4WmPmxMzh7q/v5tfkX62OozzA1lNbGTF/BHPvnEvLmi2tjlNkWkBKqGt4V97u+Ta9Z/YmMSXR6jiqDDt84TAD/ncAH/X/yPIBgopLD2FKYWTUSNIvp9NzZk9WjVrlFuMzqLLlXMY5+n7Rl2c7PcudN95pdZxi0wJSSo+0f4Tfsn+j58ye/DTqJ2pULGhUR6X+6FzGOXrO7MnQZkN5vMPjVscpET2EcYBnOz3Lnc3vpPfM3qRkplgdR5UBZ387S/cZ3endsDf/7vFvq+OUmBYQB/ln13/SvUF3es3sxdnfzlodR7mx0+mn6fZ5NwY2GcibPd+0dFT10nJ6YykR6ZanqdRWEbl0dWR2EflMRBLzvBZVmjxWEhHe6f0OfRr1ofNnnTmWeszqSMoNnUo/RbfPuxHTPIbXu71eposHuKCxlDFmxdWmUtj6wGQAP+aZ5Zk8Tae2ljKPpUSECT0mMPamsdzy6S16iVf9QWJKIl0+68KIFiN4teurZb54gOsbS8UA3xtjMkq5Xbf21M1P8c+u/6TrZ10/9yMCAAALkElEQVRJOOncoRdV2bDxxEY6xXbisfaP8XKXl62O4zCuaix11XBg1jXTJojIdhF57+ro7fkRkXH25lQJZ8+6/zmGUVGjmDxgMv2+7KfDInq5b/Z8w4D/HcCUgVN4tP2jVsdxLGPMdR/AMmBnPo/B2DrL5Z035TrrqQ2cBfyumSZAALY9mH8UlscYQ5s2bUxZsebIGlN7Ym3z9pq3TW5urtVxlAvl5uaad9e+a+q+U9cknEiwOk6xAAmmCH+Lhd4HYozpWdBrInJaRGobY5Ku11jKbhjwjTEmO8+6k+xPs0TkU8DjvubaKawTG8ZsYMicIWw9vZVpA6dRwa+C1bGUk2XlZPFk3JOsOrKKtX9d67GdDp3eWCqPEVxz+JKnq51gO3+ys5R53FL9KvVZ88AaAG799Fa9QuPhDl84zK2f3sqp9FP8PPpnjy0e4JrGUohIOFAf+Oma5b+098ndAYQA7j8AQglV8KvAF0O/4K4b76LDtA4sPbjU6kjKCRbtW0SHaR0Y3mI484fNp0r5KlZHciptLGWB+MR4Ri4YybDmw/h3j3+77XiXquhycnN4cfmLzNo5i9kxs4muH211pFLRxlJurHuD7mx9cCuJFxLpMK0Du8/uLnwh5bb2ndtHl8+6sO30Nn558JcyXzyKQwuIRaoHVmf+sPk82v5RunzWhY83fUxZ3Bv0Zldyr/DuuneJnh7N8BuHs+SeJYQEhlgdy6X0EMYN7Du3j3u+vocg/yA+ue0Ttx6FW9nsTd7L6IWj8fXxJXZQLI2qNbI6kkPpIUwZ0qR6E9b9dR1Dmg7hlthb+MeKf3Ap55LVsVQ+sq9kM3HtRDrFdmL4jcNZMXKFxxWP4tAC4iZ8fXwZ33E82/62jd1nd9Pyk5YsO7TM6lgqjx8O/ECrSa1YemgpG8Zs4LEOj+Ej3v0npAMKuZm6lesyb9g8Fu9bzNjvxtKuTjsmdJ9ARPUIq6N5rX3n9vFU3FPsPbeX9/q8x20Rt3nEF+EcwbvLpxu7rclt7Hp4F1G1orh5+s08vPhhTqWfsjqWV0nJTOHpH58meno0XcO7suvhXQxoMkCLRx5aQNxYoF8gL9z6Ansf3UugXyA3fnwjL8W/pO01nexcxjlein+Jxv9tzMWsi+x6eBdPRz+Nfzl/q6O5HS0gZUD1wOpM7D2RLQ9u4UTaCSL+G8GrK1/Vkc8cLDkjmReWv0CTD5twOv00CWMTmDJwCjWDalodzW1pASlDwqqE8engT1n1wCpOXDxBkw+b8PDihzlw/oDV0cq0o6lHeW7pczT9sCnnM8+zedxmpg6aSoPgBlZHc3taQMqgZiHNmDpoKnse2UO1CtW4efrNxMyNYd2xdXozWhEZY1h2aBlD5wyl9eTWXMq5xJYHtzBpwCTCq4ZbHa/M0BvJPED65XRit8Ty343/xb+cP6OjRnNvq3t11zsfqZdSmbFtBh8nfIyfjx+PtHuEe1rZbuJT/6eoN5JpAfEgxhjWHF1D7NZYvtnzDd0adGN01Gj6RfQrE42anSUzO5Ml+5cwe9dsfjz4I30b9+XRdo9yS9gtekWlAFpAvFxaVhpzd80ldmss+87tY0CTAQxuOpjejXoT6BdodTyny76SzbJDy5i1cxbf7fuOm2rfxIgWI7g98naqVahmdTy3pwVE/e7IhSMs3LuQBXsXsOnEJro16MaQpkPoH9Hfow5zDl84TNyBOOIOxrHi8AqahTRjRIsR3Nn8TmpXqm11vDLFJQVERO4EXgUigfbGmHz/qkWkL/AfoBwwzRhzdeChBsBsoBrwC3CfMeZyYdvVAlJy5zPPs2T/Ehb8uoBlh5ZRu1JtOod1pkt4Fzrf0Jl6letZHbHIktKS2HhiI8sTlxN3MI6UzBR6N+pNn0Z96NWoF7WCalkdscxyVQGJBHKBycDT+RUQESkH7MM2YtlxYBMwwhizW0TmAl8bY2aLyCRgmzHmk8K2qwXEMa7kXmH76e2sOrKKVUdXserIKir5VyK6fjStaraiZY2WtKzZkrqV6lp+ruDCpQtsPrmZTSc3sfHERjad3ERGdgbt6rSjyw1d6NO4D1G1orz+uymO4tJDGBFZScEF5GbgVWNMH/vvz9tfehPbKO21jDE51853PVpAnMMYw57kPaw/vp4dp3ew/cx2dpzeQU5uDi1qtKBFjRbcUOUG6lWuR/0q9alfuT51K9d1yB2aWTlZJGckc/q30xw4f4D95/az/7z9cW4/GdkZtK7dmnZ12tG+bnva1WlHw+CGlhc2T1XUAuKKU/N1gbyjCB8HOgDVsbWFyMkzva4L8qgCiAjNQ5vTPLT5H6afTj/NjjM72H12N8dSj7Hl1BaOXzzOsYvHSEpLIrhCMMHlgwnyDyLIP4hKAZVsz/2CKOdTjpzcnN8f2bnZ5OTmkJmdSXJGMmczzpKckUxWThahFUMJDQylUbVGRFSL4NawWxndejQR1SKoFVRLi4UbKrSAiMgyIL+DyReNMdcbhf33VeQzzVxnekE5xgHjAMLCPHeUa3dUM6gmNYNq0rPhnzt8XMm9wpnfzpCalUr65XTSstJIv5xue345jVyTi6+P758eFXwrEBIYQkhgCKEVQ6nkX0kLRBlUqr4wRXQc24jsV9UDTgLJQFUR8bXvhVydXlCOKcAUsB3ClDKTcpByPuWoXam2XuXwUq4447QJiBCRBiLij6295UJ796sV2PrlQuF9ZZRSbqZUBUREhorIceBmYLGIxNmn1xGRJQD2vYtHgThgDzDXGLPLvorngKdE5AC2cyLTS5NHKeVaeiOZUupPdFBlpZTTaQFRSpWYFhClVIlpAVFKlZgWEKVUiZXJqzAichY4UoRZQ7DdsFaWecJ7AM94H970Hm4wxoQWNlOZLCBFJSIJRbkU5c484T2AZ7wPfQ9/pocwSqkS0wKilCoxTy8gU6wO4ACe8B7AM96HvodrePQ5EKWUc3n6HohSyok8qoCIyJ0isktEckWkwDPNItJXRPaKyAER+bsrMxZGRKqJyFIR2W//GVzAfFdEZKv9sdDVOfNT2OcqIgEiMsf++gYRCXd9ysIV4X2MEpGzeT7/MVbkvB4RiRWRMyKys4DXRUQ+sL/H7SJyU4k2ZIzxmAe20eGbAiuBtgXMUw44CDQE/IFtQHOrs+fJ9zbwd/vzvwNvFTBfutVZi/u5Ag8Dk+zPhwNzrM5dwvcxCvjQ6qyFvI/OwE3AzgJe7w98j21kwI7AhpJsx6P2QIwxe4wxewuZrT1wwBhzyNhaSMwGBjs/XZENBj63P/8cGGJhluIoyuea973NA3qI+41j6O7/fxSJMWYVcP46swwGZhib9dhGByz2sHIeVUCKKL9Bnt1pMOeaxpgkAPvPGgXMV15EEkRkvYi4Q5Epyuf6+zzGNtBUKraBpNxJUf//uMO+6z9PROrn87q7c8jfQZlrmOrEQZ5d5nrvoRirCTPGnBSRhkC8iOwwxhx0TMISKcrnavlnXwRFyfgdMMsYkyUif8O2V9Xd6ckcyyH/LcpcATHOG+TZZa73HkTktIjUNsYk2XcpzxSwjpP2n4fsfXlaYzt2t0pRPter8xwXEV+gCtffzbZCoe/DGHMuz69TgbdckMvRHPJ34I2HMPkO8mxxprwWYhtgGgoYaFpEgkUkwP48BOgE7HZZwvwV5XPN+95igHhjP6PnRgp9H9ecKxiEbazfsmYhcL/9akxHIPXqoXOxWH222MFnnodiq6xZwGkgzj69DrDkmjPQ+7D9i/2i1bmveQ/VgeXAfvvPavbpbbH1FQaIBnZgu0KwA/ir1bkL+lyB14BB9uflga+AA8BGoKHVmUv4Pt4Adtk//xVAM6sz5/MeZgFJQLb9b+KvwN+Av9lfF+Aj+3vcQQFXLQt76J2oSqkS88ZDGKWUg2gBUUqVmBYQpVSJaQFRSpWYFhClVIlpAVFKlZgWEKVUiWkBUUqV2P8HMT0YabcAMjcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7e1758198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize predictions\n",
    "\n",
    "sum_error = 0\n",
    "for i in range(num_test):\n",
    "    y = ((data_test[i,:,:]-np.array(mc_estimation)[i,:,:])**2)\n",
    "    sum_error = sum_error +y\n",
    "#     print(((data_test[i,:,:]-np.array(mc_estimation)[i,:,:])**2).mean())\n",
    "\n",
    "\n",
    "# print(sum_error/num_test)    \n",
    "\n",
    "k=97\n",
    "# print(mc_cost)\n",
    "# estimation_error_k = np.linalg.norm(data_test[k,:,:]-np.array(mc_estimation)[k,:,:],2)\n",
    "estimation_error_k = np.mean((data_test[k,:,:]-np.array(mc_estimation)[k,:,:])**2)\n",
    "print('Estimation error for test point k=',k,':', estimation_error_k)\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(np.array(mc_estimation)[k,:,0],np.array(mc_estimation)[k,:,1],color='b', marker='o')\n",
    "plt.plot(data_test[k,:,0],data_test[k,:,1],color='r', linewidth=4)\n",
    "t = np.linspace(0,np.pi*2,100)\n",
    "plt.plot(np.cos(t), np.sin(t), linewidth=1, color='g')\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(np.array(est_data_batch)[k,:,0],np.array(est_data_batch)[k,:,1],color='b', marker='o')\n",
    "plt.plot(data_train_batch[k,:,0],data_train_batch[k,:,1],color='r', linewidth=4)\n",
    "t = np.linspace(0,np.pi*2,100)\n",
    "plt.plot(np.cos(t), np.sin(t), linewidth=1, color='g')\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Can an RNN learn the pattern - add one - ex: 1, 2, 3   or  31, 32, 33..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Can I force the weights of the RNN to learn a pattern?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "1600000\n",
      "1700000\n",
      "1800000\n",
      "1900000\n",
      "2000000\n",
      "2100000\n",
      "2200000\n",
      "2300000\n",
      "2400000\n",
      "2500000\n",
      "2600000\n",
      "2700000\n",
      "2800000\n",
      "2900000\n",
      "3000000\n",
      "3100000\n",
      "3200000\n",
      "3300000\n",
      "3400000\n",
      "3500000\n",
      "3600000\n",
      "3700000\n",
      "3800000\n",
      "3900000\n",
      "4000000\n",
      "4100000\n",
      "4200000\n",
      "4300000\n",
      "4400000\n",
      "4500000\n",
      "4600000\n",
      "4700000\n",
      "4800000\n",
      "4900000\n",
      "5000000\n",
      "5100000\n",
      "5200000\n",
      "5300000\n",
      "5400000\n",
      "5500000\n",
      "5600000\n",
      "5700000\n",
      "5800000\n",
      "5900000\n",
      "6000000\n",
      "6100000\n",
      "6200000\n",
      "6300000\n",
      "6400000\n",
      "6500000\n",
      "6600000\n",
      "6700000\n",
      "6800000\n",
      "6900000\n",
      "7000000\n",
      "7100000\n",
      "7200000\n",
      "7300000\n",
      "7400000\n",
      "7500000\n",
      "7600000\n",
      "7700000\n",
      "7800000\n",
      "7900000\n",
      "8000000\n",
      "8100000\n",
      "8200000\n",
      "8300000\n",
      "8400000\n",
      "8500000\n",
      "8600000\n",
      "8700000\n",
      "8800000\n",
      "8900000\n",
      "9000000\n",
      "9100000\n",
      "9200000\n",
      "9300000\n",
      "9400000\n",
      "9500000\n",
      "9600000\n",
      "9700000\n",
      "9800000\n",
      "9900000\n",
      "10000000\n",
      "10100000\n",
      "10200000\n",
      "10300000\n",
      "10400000\n",
      "10500000\n",
      "10600000\n",
      "10700000\n",
      "10800000\n",
      "10900000\n",
      "11000000\n",
      "11100000\n",
      "11200000\n",
      "11300000\n",
      "11400000\n",
      "11500000\n",
      "11600000\n",
      "11700000\n",
      "11800000\n",
      "11900000\n",
      "12000000\n",
      "12100000\n",
      "12200000\n",
      "12300000\n",
      "12400000\n",
      "12500000\n",
      "12600000\n",
      "12700000\n",
      "12800000\n",
      "12900000\n",
      "13000000\n",
      "13100000\n",
      "13200000\n",
      "13300000\n",
      "13400000\n",
      "13500000\n",
      "13600000\n",
      "13700000\n",
      "13800000\n",
      "13900000\n",
      "14000000\n",
      "14100000\n",
      "14200000\n",
      "14300000\n",
      "14400000\n",
      "14500000\n",
      "14600000\n",
      "14700000\n",
      "14800000\n",
      "14900000\n",
      "15000000\n",
      "15100000\n",
      "15200000\n",
      "15300000\n",
      "15400000\n",
      "15500000\n",
      "15600000\n",
      "15700000\n",
      "15800000\n",
      "15900000\n",
      "16000000\n",
      "16100000\n",
      "16200000\n",
      "16300000\n",
      "16400000\n",
      "16500000\n",
      "16600000\n",
      "16700000\n",
      "16800000\n",
      "16900000\n",
      "17000000\n",
      "17100000\n",
      "17200000\n",
      "17300000\n",
      "17400000\n",
      "17500000\n",
      "17600000\n",
      "17700000\n",
      "17800000\n",
      "17900000\n",
      "18000000\n",
      "18100000\n",
      "18200000\n",
      "18300000\n",
      "18400000\n",
      "18500000\n",
      "18600000\n",
      "18700000\n",
      "18800000\n",
      "18900000\n",
      "19000000\n",
      "19100000\n",
      "19200000\n",
      "19300000\n",
      "19400000\n",
      "19500000\n",
      "19600000\n",
      "19700000\n",
      "19800000\n",
      "19900000\n",
      "20000000\n",
      "20100000\n",
      "20200000\n",
      "20300000\n",
      "20400000\n",
      "20500000\n",
      "20600000\n",
      "20700000\n",
      "20800000\n",
      "20900000\n",
      "21000000\n",
      "21100000\n",
      "21200000\n",
      "21300000\n",
      "21400000\n",
      "21500000\n",
      "21600000\n",
      "21700000\n",
      "21800000\n",
      "21900000\n",
      "22000000\n",
      "22100000\n",
      "22200000\n",
      "22300000\n",
      "22400000\n",
      "22500000\n",
      "22600000\n",
      "22700000\n",
      "22800000\n",
      "22900000\n",
      "23000000\n",
      "23100000\n",
      "23200000\n",
      "23300000\n",
      "23400000\n",
      "23500000\n",
      "23600000\n",
      "23700000\n",
      "23800000\n",
      "23900000\n",
      "24000000\n",
      "24100000\n",
      "24200000\n",
      "24300000\n",
      "24400000\n",
      "24500000\n",
      "24600000\n",
      "24700000\n",
      "24800000\n",
      "24900000\n",
      "25000000\n",
      "25100000\n",
      "25200000\n",
      "25300000\n",
      "25400000\n",
      "25500000\n",
      "25600000\n",
      "25700000\n",
      "25800000\n",
      "25900000\n",
      "26000000\n",
      "26100000\n",
      "26200000\n",
      "26300000\n",
      "26400000\n",
      "26500000\n",
      "26600000\n",
      "26700000\n",
      "26800000\n",
      "26900000\n",
      "27000000\n",
      "27100000\n",
      "27200000\n",
      "27300000\n",
      "27400000\n",
      "27500000\n",
      "27600000\n",
      "27700000\n",
      "27800000\n",
      "27900000\n",
      "28000000\n",
      "28100000\n",
      "28200000\n",
      "28300000\n",
      "28400000\n",
      "28500000\n",
      "28600000\n",
      "28700000\n",
      "28800000\n",
      "28900000\n",
      "29000000\n",
      "29100000\n",
      "29200000\n",
      "29300000\n",
      "29400000\n",
      "29500000\n",
      "29600000\n",
      "29700000\n",
      "29800000\n",
      "29900000\n",
      "Data generation complete.\n"
     ]
    }
   ],
   "source": [
    "# create training data\n",
    "\n",
    "import numpy as np\n",
    "import scipy.signal as sig\n",
    "from numpy import linalg as LA\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create training and test data for the NN\n",
    "\n",
    "epochs = 30000\n",
    "batch_size = 1000\n",
    "\n",
    "# number of random channels that will be trained and tested on\n",
    "num_train=epochs*batch_size\n",
    "num_test=batch_size\n",
    "\n",
    "data_length = 15\n",
    "increase = 2 # integer to add\n",
    "\n",
    "\n",
    "# QPSK\n",
    "data_train = np.zeros((num_train, data_length))\n",
    "data_test = np.zeros((num_test, data_length))\n",
    "start_train = np.zeros((num_train,1))\n",
    "start_test = np.zeros((num_test,1))\n",
    "\n",
    "norm_train = np.zeros((num_train,1))\n",
    "norm_test = np.zeros((num_test,1))\n",
    "\n",
    "for i in range(num_train):\n",
    "    # first choose a random starting integer\n",
    "    random_start = np.random.randint(low=0, high=10, size=1)\n",
    "#     random_start = 0\n",
    "\n",
    "    data_train[i,0]= random_start+increase\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_train[i,j]= data_train[i,j-1] + increase\n",
    "        \n",
    "    # normalize the data: x - min(x)/(max(x)-min(x))\n",
    "#     data_train[i,:] = (data_train[i,:]-random_start)/(data_train[i,data_length-1]-random_start)\n",
    "    \n",
    "    start_train[i] = data_train[i,0]\n",
    "    \n",
    "    if i % 100000 == 0:\n",
    "        print(i)\n",
    "    \n",
    "\n",
    "for i in range(num_test):\n",
    "    # first choose a random starting integer\n",
    "    random_start = np.random.randint(low=0, high=10, size=1)\n",
    "#     random_start = 0\n",
    "\n",
    "    # real part of data\n",
    "    data_test[i,0]= random_start+increase\n",
    "    \n",
    "    # rotate data according to omega\n",
    "    for j in range(1,data_length):\n",
    "        data_test[i,j]= data_test[i,j-1] + increase\n",
    "        \n",
    "    # normalize the data: x - min(x)/(max(x)-min(x))\n",
    "#     data_test[i,:] = (data_test[i,:]-random_start)/(data_test[i,data_length-1]-random_start)\n",
    "    \n",
    "    start_test[i] = data_test[i,0]\n",
    "    \n",
    "print(\"Data generation complete.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/laura/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "delta = -0.2\n",
    "\n",
    "w1 = ([[1+delta]])\n",
    "b1 = ([[increase+delta]])\n",
    "\n",
    "w1_init = tf.constant_initializer(w1)\n",
    "b1_init = tf.constant_initializer(b1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import linalg as LA\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# fix random seed for reproducibility\n",
    "np.random.seed()\n",
    "\n",
    "# tf.reset_default_graph()\n",
    "\n",
    "learning_rate = 0.1\n",
    "decay = 1 - 1*1e-10\n",
    "\n",
    "state_size = 1 # state size\n",
    "output_size = 1 # size of the output\n",
    "\n",
    "batch_data_placeholder = tf.placeholder(tf.float32, [None, data_length])\n",
    "batch_start_placeholder = tf.placeholder(tf.float32, [None, 1])\n",
    "\n",
    "\n",
    "init_state = tf.placeholder(tf.float32, [None, state_size])\n",
    "\n",
    "# Unpack columns\n",
    "data = tf.unstack(batch_data_placeholder, axis=1)\n",
    "\n",
    "# Forward pass\n",
    "current_state = init_state\n",
    "est_data_series = []\n",
    "layer_1_series=[]\n",
    "for current_data in data:\n",
    "        \n",
    "    layer_1 = tf.layers.dense(\n",
    "        current_state, 1, activation=tf.identity, use_bias=True)\n",
    "#         kernel_initializer=w1_init, bias_initializer=b1_init, name='onelayer', reuse=tf.AUTO_REUSE)\n",
    "    \n",
    "    layer_1_series.append(tf.reshape(layer_1,[batch_size,1]))\n",
    "    \n",
    "    next_state = tf.reshape(layer_1,[batch_size,1])\n",
    "    \n",
    "    est_data_series.append(tf.reshape(layer_1,[batch_size,1]))\n",
    "#     print(est_data_series)\n",
    "#     print(np.array(est_data_series).shape)\n",
    "    current_state = next_state\n",
    "\n",
    "est_data_series_2 = tf.transpose(tf.reshape(est_data_series,[data_length,batch_size]))\n",
    "    \n",
    "estimation_cost = tf.losses.mean_squared_error(batch_data_placeholder,est_data_series_2)\n",
    "\n",
    "# print(tf.gradients(est_data_series_2,layer_1))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate).minimize(estimation_cost)\n",
    "                                                                                  \n",
    "# finally setup the initialisation operator\n",
    "init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Train Cost 497.2836608886719, Test Cost: 498.1287536621094\n",
      "Epoch 100, Train Cost 136.08522033691406, Test Cost: 135.58348083496094\n",
      "Epoch 200, Train Cost 123.97468566894531, Test Cost: 117.50873565673828\n",
      "Epoch 300, Train Cost 113.59136962890625, Test Cost: 112.80001068115234\n",
      "Epoch 400, Train Cost 108.90225219726562, Test Cost: 108.41967010498047\n",
      "Epoch 500, Train Cost 115.00662994384766, Test Cost: 114.6864013671875\n",
      "Epoch 600, Train Cost 103.79845428466797, Test Cost: 103.67005157470703\n",
      "Epoch 700, Train Cost 104.32888793945312, Test Cost: 102.27153778076172\n",
      "Epoch 800, Train Cost 99.05237579345703, Test Cost: 98.45833587646484\n",
      "Epoch 900, Train Cost 97.91514587402344, Test Cost: 97.70852661132812\n",
      "Epoch 1000, Train Cost 96.00230407714844, Test Cost: 95.5372543334961\n",
      "Epoch 1100, Train Cost 91.02989959716797, Test Cost: 90.8856201171875\n",
      "Epoch 1200, Train Cost 86.36434936523438, Test Cost: 85.79611206054688\n",
      "Epoch 1300, Train Cost 67.01500701904297, Test Cost: 66.7822036743164\n",
      "Epoch 1400, Train Cost 59.03203582763672, Test Cost: 58.122867584228516\n",
      "Epoch 1500, Train Cost 54.0272102355957, Test Cost: 54.16688919067383\n",
      "Epoch 1600, Train Cost 7.525563716888428, Test Cost: 7.546927452087402\n",
      "Epoch 1700, Train Cost 7.210836410522461, Test Cost: 7.419524192810059\n",
      "Epoch 1800, Train Cost 7.079677104949951, Test Cost: 7.208840847015381\n",
      "Epoch 1900, Train Cost 7.18778657913208, Test Cost: 6.861642360687256\n",
      "Epoch 2000, Train Cost 6.137734413146973, Test Cost: 6.325572490692139\n",
      "Epoch 2100, Train Cost 5.841503620147705, Test Cost: 5.848447799682617\n",
      "Epoch 2200, Train Cost 5.809720993041992, Test Cost: 5.556923866271973\n",
      "Epoch 2300, Train Cost 5.3360595703125, Test Cost: 5.368405342102051\n",
      "Epoch 2400, Train Cost 5.567646026611328, Test Cost: 5.162746906280518\n",
      "Epoch 2500, Train Cost 4.990287780761719, Test Cost: 4.969202518463135\n",
      "Epoch 2600, Train Cost 4.949009418487549, Test Cost: 4.692844867706299\n",
      "Epoch 2700, Train Cost 4.704481601715088, Test Cost: 4.453836441040039\n",
      "Epoch 2800, Train Cost 4.0625, Test Cost: 4.247446060180664\n",
      "Epoch 2900, Train Cost 4.29231595993042, Test Cost: 4.136693477630615\n",
      "Epoch 3000, Train Cost 3.9698824882507324, Test Cost: 3.9045698642730713\n",
      "Epoch 3100, Train Cost 3.8848843574523926, Test Cost: 3.7320845127105713\n",
      "Epoch 3200, Train Cost 3.5795984268188477, Test Cost: 3.5917916297912598\n",
      "Epoch 3300, Train Cost 3.47597599029541, Test Cost: 3.504786491394043\n",
      "Epoch 3400, Train Cost 3.384671926498413, Test Cost: 3.3260257244110107\n",
      "Epoch 3500, Train Cost 3.1099023818969727, Test Cost: 3.2157206535339355\n",
      "Epoch 3600, Train Cost 3.0439839363098145, Test Cost: 3.039738893508911\n",
      "Epoch 3700, Train Cost 2.9194867610931396, Test Cost: 2.8898847103118896\n",
      "Epoch 3800, Train Cost 2.8884615898132324, Test Cost: 2.6506285667419434\n",
      "Epoch 3900, Train Cost 2.4585626125335693, Test Cost: 2.417816162109375\n",
      "Epoch 4000, Train Cost 2.2094926834106445, Test Cost: 2.218806266784668\n",
      "Epoch 4100, Train Cost 2.0100131034851074, Test Cost: 1.9349019527435303\n",
      "Epoch 4200, Train Cost 1.7099025249481201, Test Cost: 1.6418249607086182\n",
      "Epoch 4300, Train Cost 1.4037914276123047, Test Cost: 1.3506537675857544\n",
      "Epoch 4400, Train Cost 1.150673508644104, Test Cost: 1.1256896257400513\n",
      "Epoch 4500, Train Cost 0.7879822850227356, Test Cost: 0.7761284112930298\n",
      "Epoch 4600, Train Cost 0.620469331741333, Test Cost: 0.6109572052955627\n",
      "Epoch 4700, Train Cost 0.6622094511985779, Test Cost: 0.7374264001846313\n",
      "Epoch 4800, Train Cost 0.2704979479312897, Test Cost: 0.2788670063018799\n",
      "Epoch 4900, Train Cost 0.18580162525177002, Test Cost: 0.18046990036964417\n",
      "Epoch 5000, Train Cost 0.2612644135951996, Test Cost: 0.36315175890922546\n",
      "Epoch 5100, Train Cost 0.12658560276031494, Test Cost: 0.10970719158649445\n",
      "Epoch 5200, Train Cost 0.10498063266277313, Test Cost: 0.10124910622835159\n",
      "Epoch 5300, Train Cost 0.22686882317066193, Test Cost: 0.2973065674304962\n",
      "Epoch 5400, Train Cost 0.10585006326436996, Test Cost: 0.13407231867313385\n",
      "Epoch 5500, Train Cost 0.05580272525548935, Test Cost: 0.05546839162707329\n",
      "Epoch 5600, Train Cost 0.11834114789962769, Test Cost: 0.1425139307975769\n",
      "Epoch 5700, Train Cost 0.051328033208847046, Test Cost: 0.04514484480023384\n",
      "Epoch 5800, Train Cost 0.07401331514120102, Test Cost: 0.07782096415758133\n",
      "Epoch 5900, Train Cost 0.06942952424287796, Test Cost: 0.08250685036182404\n",
      "Epoch 6000, Train Cost 1.4545807838439941, Test Cost: 2.0675010681152344\n",
      "Epoch 6100, Train Cost 0.03596167266368866, Test Cost: 0.044179316610097885\n",
      "Epoch 6200, Train Cost 0.0353914350271225, Test Cost: 0.034141071140766144\n",
      "Epoch 6300, Train Cost 0.48573771119117737, Test Cost: 0.5835754871368408\n",
      "Epoch 6400, Train Cost 0.04872377589344978, Test Cost: 0.025563079863786697\n",
      "Epoch 6500, Train Cost 0.02841782756149769, Test Cost: 0.030687399208545685\n",
      "Epoch 6600, Train Cost 0.25074225664138794, Test Cost: 0.3054903745651245\n",
      "Epoch 6700, Train Cost 0.049753427505493164, Test Cost: 0.08021088689565659\n",
      "Epoch 6800, Train Cost 0.07350678741931915, Test Cost: 0.07843495905399323\n",
      "Epoch 6900, Train Cost 0.03840629756450653, Test Cost: 0.03967633470892906\n",
      "Epoch 7000, Train Cost 0.15287604928016663, Test Cost: 0.2214575558900833\n",
      "Epoch 7100, Train Cost 0.027007125318050385, Test Cost: 0.025153497233986855\n",
      "Epoch 7200, Train Cost 0.7707079648971558, Test Cost: 0.4191860258579254\n",
      "Epoch 7300, Train Cost 0.07041139900684357, Test Cost: 0.11956796795129776\n",
      "Epoch 7400, Train Cost 0.1984626203775406, Test Cost: 0.2785729467868805\n",
      "Epoch 7500, Train Cost 0.01542514469474554, Test Cost: 0.013472074642777443\n",
      "Epoch 7600, Train Cost 0.03505688160657883, Test Cost: 0.047368716448545456\n",
      "Epoch 7700, Train Cost 0.13304777443408966, Test Cost: 0.18121403455734253\n",
      "Epoch 7800, Train Cost 0.055017415434122086, Test Cost: 0.025699829682707787\n",
      "Epoch 7900, Train Cost 0.17572273313999176, Test Cost: 0.18602345883846283\n",
      "Epoch 8000, Train Cost 0.1442263424396515, Test Cost: 0.1702658236026764\n",
      "Epoch 8100, Train Cost 0.12666060030460358, Test Cost: 0.08404490351676941\n",
      "Epoch 8200, Train Cost 0.018009912222623825, Test Cost: 0.011324760504066944\n",
      "Epoch 8300, Train Cost 0.012032178230583668, Test Cost: 0.013496783562004566\n",
      "Epoch 8400, Train Cost 0.08289340138435364, Test Cost: 0.10798811912536621\n",
      "Epoch 8500, Train Cost 0.15629830956459045, Test Cost: 0.07932725548744202\n",
      "Epoch 8600, Train Cost 0.2837318480014801, Test Cost: 0.3717130124568939\n",
      "Epoch 8700, Train Cost 0.10789884626865387, Test Cost: 0.07601125538349152\n",
      "Epoch 8800, Train Cost 0.009464343078434467, Test Cost: 0.011472986079752445\n",
      "Epoch 8900, Train Cost 0.03406906500458717, Test Cost: 0.029777266085147858\n",
      "Epoch 9000, Train Cost 0.021665992215275764, Test Cost: 0.015004687942564487\n",
      "Epoch 9100, Train Cost 0.08461199700832367, Test Cost: 0.04044521972537041\n",
      "Epoch 9200, Train Cost 0.1268835961818695, Test Cost: 0.11676409095525742\n",
      "Epoch 9300, Train Cost 0.07601198554039001, Test Cost: 0.06997571140527725\n",
      "Epoch 9400, Train Cost 0.3886439800262451, Test Cost: 0.13387614488601685\n",
      "Epoch 9500, Train Cost 0.3731088638305664, Test Cost: 0.12188888341188431\n",
      "Epoch 9600, Train Cost 0.22358004748821259, Test Cost: 0.30371588468551636\n",
      "Epoch 9700, Train Cost 0.02680794708430767, Test Cost: 0.021540416404604912\n",
      "Epoch 9800, Train Cost 0.050914566963911057, Test Cost: 0.05969058349728584\n",
      "Epoch 9900, Train Cost 0.0057309712283313274, Test Cost: 0.005508745554834604\n",
      "Epoch 10000, Train Cost 0.01029229536652565, Test Cost: 0.013040194287896156\n",
      "Epoch 10100, Train Cost 0.1890048384666443, Test Cost: 0.20669837296009064\n",
      "Epoch 10200, Train Cost 0.2379598617553711, Test Cost: 0.03373192250728607\n",
      "Epoch 10300, Train Cost 0.9556545615196228, Test Cost: 0.20339180529117584\n",
      "Epoch 10400, Train Cost 0.006087343208491802, Test Cost: 0.005554892588406801\n",
      "Epoch 10500, Train Cost 0.9000842571258545, Test Cost: 0.9541850090026855\n",
      "Epoch 10600, Train Cost 0.8175554871559143, Test Cost: 0.8960558176040649\n",
      "Epoch 10700, Train Cost 1.0224369764328003, Test Cost: 0.9462862014770508\n",
      "Epoch 10800, Train Cost 0.020085809752345085, Test Cost: 0.0068778395652771\n",
      "Epoch 10900, Train Cost 0.034082792699337006, Test Cost: 0.021526213735342026\n",
      "Epoch 11000, Train Cost 0.02460133098065853, Test Cost: 0.029439806938171387\n",
      "Epoch 11100, Train Cost 0.02345876209437847, Test Cost: 0.022225407883524895\n",
      "Epoch 11200, Train Cost 0.24458667635917664, Test Cost: 0.19467128813266754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11300, Train Cost 0.027362443506717682, Test Cost: 0.022262349724769592\n",
      "Epoch 11400, Train Cost 0.021239690482616425, Test Cost: 0.026196932420134544\n",
      "Epoch 11500, Train Cost 0.020045991986989975, Test Cost: 0.018321912735700607\n",
      "Epoch 11600, Train Cost 0.04239135608077049, Test Cost: 0.0393839105963707\n",
      "Epoch 11700, Train Cost 0.04508469998836517, Test Cost: 0.1242595762014389\n",
      "Epoch 11800, Train Cost 0.018392574042081833, Test Cost: 0.042219873517751694\n",
      "Epoch 11900, Train Cost 0.02632269263267517, Test Cost: 0.032362814992666245\n",
      "Epoch 12000, Train Cost 0.005750098265707493, Test Cost: 0.0052471160888671875\n",
      "Epoch 12100, Train Cost 0.005374656058847904, Test Cost: 0.005381301511079073\n",
      "Epoch 12200, Train Cost 0.00960023794323206, Test Cost: 0.010985884815454483\n",
      "Epoch 12300, Train Cost 0.2701810598373413, Test Cost: 0.3076518177986145\n",
      "Epoch 12400, Train Cost 0.018656514585018158, Test Cost: 0.009119023568928242\n",
      "Epoch 12500, Train Cost 0.0049590435810387135, Test Cost: 0.00474929716438055\n",
      "Epoch 12600, Train Cost 0.05214466154575348, Test Cost: 0.07146254926919937\n",
      "Epoch 12700, Train Cost 0.256903737783432, Test Cost: 0.3776581287384033\n",
      "Epoch 12800, Train Cost 0.011600136756896973, Test Cost: 0.012945841997861862\n",
      "Epoch 12900, Train Cost 0.004946226719766855, Test Cost: 0.004536766558885574\n",
      "Epoch 13000, Train Cost 0.44091275334358215, Test Cost: 0.7088634967803955\n",
      "Epoch 13100, Train Cost 0.005742174107581377, Test Cost: 0.00869030598551035\n",
      "Epoch 13200, Train Cost 0.038107674568891525, Test Cost: 0.03800571709871292\n",
      "Epoch 13300, Train Cost 0.14392627775669098, Test Cost: 0.0493113249540329\n",
      "Epoch 13400, Train Cost 0.027266336604952812, Test Cost: 0.03936508670449257\n",
      "Epoch 13500, Train Cost 0.06936637312173843, Test Cost: 0.1393108069896698\n",
      "Epoch 13600, Train Cost 0.14985324442386627, Test Cost: 0.09273841977119446\n",
      "Epoch 13700, Train Cost 0.12842988967895508, Test Cost: 0.34026727080345154\n",
      "Epoch 13800, Train Cost 0.9169396162033081, Test Cost: 0.9412552714347839\n",
      "Epoch 13900, Train Cost 0.06094781309366226, Test Cost: 0.09466158598661423\n",
      "Epoch 14000, Train Cost 0.06750018149614334, Test Cost: 0.07713700085878372\n",
      "Epoch 14100, Train Cost 0.0072971535846591, Test Cost: 0.01850518397986889\n",
      "Epoch 14200, Train Cost 0.04044605791568756, Test Cost: 0.022075491026043892\n",
      "Epoch 14300, Train Cost 0.021189620718359947, Test Cost: 0.02997676655650139\n",
      "Epoch 14400, Train Cost 0.16742470860481262, Test Cost: 0.26232150197029114\n",
      "Epoch 14500, Train Cost 0.10085505992174149, Test Cost: 0.043259065598249435\n",
      "Epoch 14600, Train Cost 0.06278310716152191, Test Cost: 0.04188833758234978\n",
      "Epoch 14700, Train Cost 0.02840033359825611, Test Cost: 0.02070545218884945\n",
      "Epoch 14800, Train Cost 0.004173692781478167, Test Cost: 0.004335632082074881\n",
      "Epoch 14900, Train Cost 0.01722443476319313, Test Cost: 0.014459342695772648\n",
      "Epoch 15000, Train Cost 1.347693920135498, Test Cost: 1.2926594018936157\n",
      "Epoch 15100, Train Cost 0.20064222812652588, Test Cost: 0.3321470022201538\n",
      "Epoch 15200, Train Cost 0.013400107622146606, Test Cost: 0.01932317204773426\n",
      "Epoch 15300, Train Cost 0.3679487109184265, Test Cost: 0.22196581959724426\n",
      "Epoch 15400, Train Cost 0.11127407103776932, Test Cost: 0.0905534029006958\n",
      "Epoch 15500, Train Cost 0.0052696168422698975, Test Cost: 0.004793860018253326\n",
      "Epoch 15600, Train Cost 0.24913440644741058, Test Cost: 0.24462366104125977\n",
      "Epoch 15700, Train Cost 0.33894169330596924, Test Cost: 0.23690365254878998\n",
      "Epoch 15800, Train Cost 0.08071738481521606, Test Cost: 0.09575948864221573\n",
      "Epoch 15900, Train Cost 0.14262858033180237, Test Cost: 0.10347294062376022\n",
      "Epoch 16000, Train Cost 0.005462935660034418, Test Cost: 0.0043392782099545\n",
      "Epoch 16100, Train Cost 0.48998939990997314, Test Cost: 0.5373570322990417\n",
      "Epoch 16200, Train Cost 0.12577848136425018, Test Cost: 0.04879225417971611\n",
      "Epoch 16300, Train Cost 0.004897891078144312, Test Cost: 0.004705751780420542\n",
      "Epoch 16400, Train Cost 0.10263874381780624, Test Cost: 0.15917716920375824\n",
      "Epoch 16500, Train Cost 0.004070851020514965, Test Cost: 0.003838323289528489\n",
      "Epoch 16600, Train Cost 0.011377057060599327, Test Cost: 0.013018975034356117\n",
      "Epoch 16700, Train Cost 0.1214490681886673, Test Cost: 0.08992486447095871\n",
      "Epoch 16800, Train Cost 0.10548803210258484, Test Cost: 0.14110516011714935\n",
      "Epoch 16900, Train Cost 0.0052947998046875, Test Cost: 0.045956552028656006\n",
      "Epoch 17000, Train Cost 0.4279858469963074, Test Cost: 0.14113393425941467\n",
      "Epoch 17100, Train Cost 0.004752448759973049, Test Cost: 0.00513864541426301\n",
      "Epoch 17200, Train Cost 0.012665590271353722, Test Cost: 0.016193419694900513\n",
      "Epoch 17300, Train Cost 0.05404506251215935, Test Cost: 0.12929482758045197\n",
      "Epoch 17400, Train Cost 0.28028982877731323, Test Cost: 0.20490118861198425\n",
      "Epoch 17500, Train Cost 0.2257857471704483, Test Cost: 0.22431522607803345\n",
      "Epoch 17600, Train Cost 0.004831135738641024, Test Cost: 0.004544881638139486\n",
      "Epoch 17700, Train Cost 0.005890778731554747, Test Cost: 0.005775115452706814\n",
      "Epoch 17800, Train Cost 0.018468327820301056, Test Cost: 0.021299414336681366\n",
      "Epoch 17900, Train Cost 0.014292159117758274, Test Cost: 0.012409687042236328\n",
      "Epoch 18000, Train Cost 0.006537027191370726, Test Cost: 0.007197275757789612\n",
      "Epoch 18100, Train Cost 0.008450424298644066, Test Cost: 0.011982063762843609\n",
      "Epoch 18200, Train Cost 0.005208038259297609, Test Cost: 0.004345763474702835\n",
      "Epoch 18300, Train Cost 0.008048848249018192, Test Cost: 0.009089515544474125\n",
      "Epoch 18400, Train Cost 0.004729765933007002, Test Cost: 0.005306553095579147\n",
      "Epoch 18500, Train Cost 0.005568003747612238, Test Cost: 0.006222529336810112\n",
      "Epoch 18600, Train Cost 0.010159224271774292, Test Cost: 0.009928178042173386\n",
      "Epoch 18700, Train Cost 0.013962846249341965, Test Cost: 0.04152155667543411\n",
      "Epoch 18800, Train Cost 0.22027260065078735, Test Cost: 0.30592039227485657\n",
      "Epoch 18900, Train Cost 0.569444477558136, Test Cost: 0.17222461104393005\n",
      "Epoch 19000, Train Cost 0.021521752700209618, Test Cost: 0.026789095252752304\n",
      "Epoch 19100, Train Cost 0.046643491834402084, Test Cost: 0.059405770152807236\n",
      "Epoch 19200, Train Cost 0.012541953474283218, Test Cost: 0.015977321192622185\n",
      "Epoch 19300, Train Cost 0.04856552928686142, Test Cost: 0.0623626634478569\n",
      "Epoch 19400, Train Cost 0.014670762233436108, Test Cost: 0.02228540927171707\n",
      "Epoch 19500, Train Cost 0.5965268015861511, Test Cost: 0.41798996925354004\n",
      "Epoch 19600, Train Cost 0.027080662548542023, Test Cost: 0.038516853004693985\n",
      "Epoch 19700, Train Cost 0.006298128515481949, Test Cost: 0.068930983543396\n",
      "Epoch 19800, Train Cost 0.7190048098564148, Test Cost: 1.0926051139831543\n",
      "Epoch 19900, Train Cost 0.024601617828011513, Test Cost: 0.065358966588974\n",
      "Epoch 20000, Train Cost 0.13348282873630524, Test Cost: 0.13809862732887268\n",
      "Epoch 20100, Train Cost 0.007775514852255583, Test Cost: 0.010466951876878738\n",
      "Epoch 20200, Train Cost 0.2832318842411041, Test Cost: 0.2403542548418045\n",
      "Epoch 20300, Train Cost 0.035978514701128006, Test Cost: 0.05160410702228546\n",
      "Epoch 20400, Train Cost 0.5339020490646362, Test Cost: 0.6210669279098511\n",
      "Epoch 20500, Train Cost 0.5571851134300232, Test Cost: 1.065220594406128\n",
      "Epoch 20600, Train Cost 0.27408114075660706, Test Cost: 0.2595628798007965\n",
      "Epoch 20700, Train Cost 0.030553894117474556, Test Cost: 0.03824780881404877\n",
      "Epoch 20800, Train Cost 0.006194773130118847, Test Cost: 0.008893879130482674\n",
      "Epoch 20900, Train Cost 0.03004486858844757, Test Cost: 0.03482629358768463\n",
      "Epoch 21000, Train Cost 0.33029302954673767, Test Cost: 0.4421716332435608\n",
      "Epoch 21100, Train Cost 0.10960043966770172, Test Cost: 0.05348287522792816\n",
      "Epoch 21200, Train Cost 0.2417580485343933, Test Cost: 0.1240122839808464\n",
      "Epoch 21300, Train Cost 0.054797932505607605, Test Cost: 0.08156292885541916\n",
      "Epoch 21400, Train Cost 1.4864715337753296, Test Cost: 1.0370720624923706\n",
      "Epoch 21500, Train Cost 0.4218311309814453, Test Cost: 0.20021717250347137\n",
      "Epoch 21600, Train Cost 0.09471796452999115, Test Cost: 0.10949870198965073\n",
      "Epoch 21700, Train Cost 0.04381995648145676, Test Cost: 0.062310993671417236\n",
      "Epoch 21800, Train Cost 0.07506212592124939, Test Cost: 0.009780715219676495\n",
      "Epoch 21900, Train Cost 0.06640657782554626, Test Cost: 0.027630873024463654\n",
      "Epoch 22000, Train Cost 0.10156530141830444, Test Cost: 0.04844927787780762\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22100, Train Cost 0.008395788259804249, Test Cost: 0.00879527535289526\n",
      "Epoch 22200, Train Cost 0.0065544904209673405, Test Cost: 0.005811460316181183\n",
      "Epoch 22300, Train Cost 0.006170530337840319, Test Cost: 0.005832333583384752\n",
      "Epoch 22400, Train Cost 0.026322076097130775, Test Cost: 0.027700409293174744\n",
      "Epoch 22500, Train Cost 0.1070043295621872, Test Cost: 0.09014492481946945\n",
      "Epoch 22600, Train Cost 0.183059424161911, Test Cost: 0.2767757177352905\n",
      "Epoch 22700, Train Cost 0.12786360085010529, Test Cost: 0.1413695067167282\n",
      "Epoch 22800, Train Cost 0.06866641342639923, Test Cost: 0.08973944932222366\n",
      "Epoch 22900, Train Cost 0.10018626600503922, Test Cost: 0.13809658586978912\n",
      "Epoch 23000, Train Cost 0.01229951623827219, Test Cost: 0.0070459176786243916\n",
      "Epoch 23100, Train Cost 0.6053489446640015, Test Cost: 0.590583860874176\n",
      "Epoch 23200, Train Cost 0.3900229334831238, Test Cost: 0.3524634838104248\n",
      "Epoch 23300, Train Cost 0.10148163884878159, Test Cost: 0.19053886830806732\n",
      "Epoch 23400, Train Cost 0.03133803978562355, Test Cost: 0.0522833913564682\n",
      "Epoch 23500, Train Cost 0.3545910716056824, Test Cost: 0.5460067391395569\n",
      "Epoch 23600, Train Cost 0.03453512117266655, Test Cost: 0.01903439871966839\n",
      "Epoch 23700, Train Cost 0.03429576754570007, Test Cost: 0.007396058179438114\n",
      "Epoch 23800, Train Cost 0.07245607674121857, Test Cost: 0.07389421761035919\n",
      "Epoch 23900, Train Cost 0.006506936624646187, Test Cost: 0.0052260528318583965\n",
      "Epoch 24000, Train Cost 0.005940495058894157, Test Cost: 0.005786434281617403\n",
      "Epoch 24100, Train Cost 0.009303908795118332, Test Cost: 0.010611871257424355\n",
      "Epoch 24200, Train Cost 0.008564756251871586, Test Cost: 0.009925675578415394\n",
      "Epoch 24300, Train Cost 0.010768566280603409, Test Cost: 0.016224633902311325\n",
      "Epoch 24400, Train Cost 0.3787493109703064, Test Cost: 0.22066444158554077\n",
      "Epoch 24500, Train Cost 0.008917199447751045, Test Cost: 0.005719528067857027\n",
      "Epoch 24600, Train Cost 0.025563523173332214, Test Cost: 0.03300569951534271\n",
      "Epoch 24700, Train Cost 0.007832176983356476, Test Cost: 0.01890498958528042\n",
      "Epoch 24800, Train Cost 0.09220504015684128, Test Cost: 0.10550007969141006\n",
      "Epoch 24900, Train Cost 0.013299413956701756, Test Cost: 0.017640506848692894\n",
      "Epoch 25000, Train Cost 0.16023431718349457, Test Cost: 0.20222832262516022\n",
      "Epoch 25100, Train Cost 0.012472487986087799, Test Cost: 0.026456885039806366\n",
      "Epoch 25200, Train Cost 0.2834683656692505, Test Cost: 0.3035777509212494\n",
      "Epoch 25300, Train Cost 0.09761743992567062, Test Cost: 0.09017298370599747\n",
      "Epoch 25400, Train Cost 0.07250331342220306, Test Cost: 0.037993282079696655\n",
      "Epoch 25500, Train Cost 0.028869913890957832, Test Cost: 0.022939372807741165\n",
      "Epoch 25600, Train Cost 0.010038613341748714, Test Cost: 0.011793013662099838\n",
      "Epoch 25700, Train Cost 1.2189513444900513, Test Cost: 1.0137351751327515\n",
      "Epoch 25800, Train Cost 0.20579390227794647, Test Cost: 0.21215319633483887\n",
      "Epoch 25900, Train Cost 0.0060586752369999886, Test Cost: 0.01977556012570858\n",
      "Epoch 26000, Train Cost 0.029513277113437653, Test Cost: 0.01850198768079281\n",
      "Epoch 26100, Train Cost 0.5465273857116699, Test Cost: 0.856827437877655\n",
      "Epoch 26200, Train Cost 0.8349007964134216, Test Cost: 1.4635413885116577\n",
      "Epoch 26300, Train Cost 1.0599684715270996, Test Cost: 0.7871096730232239\n",
      "Epoch 26400, Train Cost 0.005999996326863766, Test Cost: 0.005575629882514477\n",
      "Epoch 26500, Train Cost 0.00488988496363163, Test Cost: 0.004112166352570057\n",
      "Epoch 26600, Train Cost 0.004062112420797348, Test Cost: 0.0037589131388813257\n",
      "Epoch 26700, Train Cost 0.010224413126707077, Test Cost: 0.010238658636808395\n",
      "Epoch 26800, Train Cost 0.07690145075321198, Test Cost: 0.09832627326250076\n",
      "Epoch 26900, Train Cost 0.3124355971813202, Test Cost: 0.06678035110235214\n",
      "Epoch 27000, Train Cost 0.5923227667808533, Test Cost: 0.1686745285987854\n",
      "Epoch 27100, Train Cost 0.10290199518203735, Test Cost: 0.17848965525627136\n",
      "Epoch 27200, Train Cost 0.00569905387237668, Test Cost: 0.004975099116563797\n",
      "Epoch 27300, Train Cost 0.005612946581095457, Test Cost: 0.0060005937702953815\n",
      "Epoch 27400, Train Cost 0.0068818447180092335, Test Cost: 0.006623276509344578\n",
      "Epoch 27500, Train Cost 0.09559045732021332, Test Cost: 0.13435803353786469\n",
      "Epoch 27600, Train Cost 0.011627132073044777, Test Cost: 0.011146253906190395\n",
      "Epoch 27700, Train Cost 0.007555776741355658, Test Cost: 0.0076204463839530945\n",
      "Epoch 27800, Train Cost 0.1735074520111084, Test Cost: 0.14692363142967224\n",
      "Epoch 27900, Train Cost 0.2736058831214905, Test Cost: 0.5637498497962952\n",
      "Epoch 28000, Train Cost 1.4668352603912354, Test Cost: 1.4719890356063843\n",
      "Epoch 28100, Train Cost 1.2760721445083618, Test Cost: 0.5958585143089294\n",
      "Epoch 28200, Train Cost 0.42101556062698364, Test Cost: 0.5394013524055481\n",
      "Epoch 28300, Train Cost 0.00607704883441329, Test Cost: 0.007986417040228844\n",
      "Epoch 28400, Train Cost 0.010060223750770092, Test Cost: 0.006437175907194614\n",
      "Epoch 28500, Train Cost 0.22732959687709808, Test Cost: 0.20419037342071533\n",
      "Epoch 28600, Train Cost 0.06199777498841286, Test Cost: 0.0163315050303936\n",
      "Epoch 28700, Train Cost 0.15536808967590332, Test Cost: 0.20429489016532898\n",
      "Epoch 28800, Train Cost 0.022000759840011597, Test Cost: 0.03394429013133049\n",
      "Epoch 28900, Train Cost 0.21634705364704132, Test Cost: 0.26494571566581726\n",
      "Epoch 29000, Train Cost 1.2455804347991943, Test Cost: 1.9261205196380615\n",
      "Epoch 29100, Train Cost 0.5002092123031616, Test Cost: 0.21004630625247955\n",
      "Epoch 29200, Train Cost 0.7779306769371033, Test Cost: 0.03500647842884064\n",
      "Epoch 29300, Train Cost 0.005480219144374132, Test Cost: 0.0052720000967383385\n",
      "Epoch 29400, Train Cost 0.005616188049316406, Test Cost: 0.0071353311650455\n",
      "Epoch 29500, Train Cost 0.009044451639056206, Test Cost: 0.010844619944691658\n",
      "Epoch 29600, Train Cost 0.030115120112895966, Test Cost: 0.03917169198393822\n",
      "Epoch 29700, Train Cost 0.005022863391786814, Test Cost: 0.00480404170230031\n",
      "Epoch 29800, Train Cost 0.004081983119249344, Test Cost: 0.004137548618018627\n",
      "Epoch 29900, Train Cost 0.08510812371969223, Test Cost: 0.11208907514810562\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXucXVV1+L9rJjM4IQ/IJLQoZMYH1NcIyqjF+qjGVhNEMK3POzFC+wtJqg32p3zUUZG281Njf5a0GmJqQyEZFdSARUIVU/FVH50gMCBFUJOIICYTCQkZmZCs/nHPCXfunHPuPvee1527vp/P/cydc89j7XPO3mvvtdZeW1QVwzAMw2jLWwDDMAyjGJhCMAzDMABTCIZhGIaHKQTDMAwDMIVgGIZheJhCMAzDMABTCIZxDBFREXmG932DiHwob5nyQEQWishBEWnPWxYjW0whGHUjIjtF5CEROb5i21+KyC0V/6uIjIpIW8W2vxeRfws55x+LyFGvQar8nJ1mWapR1ZWq+ndZXrMoqOpuVZ2lqkfylsXIFlMIRqPMANbU2OfJwFtinPMBr0Gq/Hy/fhGLgfW4jaJjCsFolE8A7xGREyL2WQtcJiIzGr2YiDxVRL4lIgdE5GYR+ZSIbPF++2MRub9q/50i8mrv+4tE5Psi8rCIPOgd2xlynX8Tkb/3vt9QNVo5KiLv8H57pifHPhG5R0TeVHWOK0Rkm4g8CrzSoXy3iMjficj3vDJ+XUTmV/z+ehG5yyvDLSLyrIhzqYisFJF7ReS3IvJpERHvtzYR+aCI7BKR34jI1SIy1/ut1zt2hvf/O0Tk5548vxCRUsU1LhSRu73zf01EemqV0SguphCMRhkBbgHeE7HPVuAR4B0JXO9zwA5gPvB3wPIYxx4B3u0dezawCFhd6yBVPdcfqQB/Dvwa2O6Zym72ZDoJeCuwXkSeU3H424AhYDbwXRF5m4jcUeOSbwMu8M7ZiXdvReR04PPAxcACYBtwQ5hS83gd8ELgDOBNwGu87e/wPq8EngbMAj5VfbBXxn8CFqvqbOAlwG3eb+cDHwCWevJ8x5PPaFJMIRhJ8GHgXSKyIOR3BT4EfFhEjnM435O9HnDl53gRWUi5cfuQqj6mqt8GbnAVUlV3qOoPVPVxVd0JfAZ4hevxXoN8NfBmVf0l5cZ2p6pe6Z3zVuDLlJWGz1dU9XuqelRVf6eqn1PV59W41JWq+lNVHQeuBc70tr8ZuFFVb1bVw8A/AF2UG+kwPqaqD6vqbuCbFecqAZ9U1Z+r6kHg/cBbQkZxR4HnikiXqj6oqnd52y8CPqqqd6vq48D/A860UULzYgrBaBhVvRP4KvC+iH22AbuBFQ6nfEBVT6j6PErZF/Fb77vPLlc5ReR0EfmqiPxaRB6h3IDNr3Wcd+xc4CuUldF3vM09wIsrFRflhvb3Kw79pat8Ffy64vshyr13KJf/WHlV9ah3/qc0ei7v+wzg9yoP9u71m4GVwIMicqOIPNP7uQdYV1H2fYDUkMcoMKYQjKS4FPg/RDcGHwQGgZl1XuNB4MTKqCZgYcX3RyvP7TlxK0ctVwD/A5ymqnMomzuk1kW9CKnPAd9U1c9U/PRL4FtVimuWqq6q2CfJdMIPUG6EfbkEOBX4VaPnonwfHwceqt5RVb+mqn8CnEz5/v2L99MvgYuqyt+lqv9VhzxGATCFYCSCqt4HXAP8dcQ+twCjxLP7Vx6/i7LP4jIR6RSRlwLnVuzyU+BJInKOiHRQVkCVJqrZlH0ZB71ebmXDHcUQcDxTo6m+CpwuIstEpMP7vDDK0dsg1wLniMgir3z/F3gMqKcB/jzwbs9JP4vyaOkaz/RzDBH5Pc+Rfbx3rYOUfTEAG4D3+z4TEZkrIm+sq2RGITCFYCTJ31JuOKP4IDCvxj5PlqnzEP7M++1twIspmycupWzTB0BV91N2En+Wcq/5UaAy6ug93vEHKPdyr3EqVdlZ/IfAbyvkKanqAeBPKYfUPkDZPPNxJiuhSYhISUTuCvs9ClW9BxgA/hnYS1kZnquqE3WcbhOwGfg28Avgd8C7AvZro6x4HqB8z1+B54hX1esol/cLngnuTmBxHbIYBUFsgRyjmRGRjwDPUNWBvGUxjGbHRgiGYRgGYArBMAzD8DCTkWEYhgHYCMEwDMPwaDi3TJbMnz9fe3t78xbDMAyjqdixY8deVQ3LJHCMplIIvb29jIyM5C2GYRhGUyEiTjP6zWRkGIZhAKYQDMMwDA9TCIZhGAZgCsEwDMPwMIVgGIZhAC2gEIZHh+m9vJe2y9rovbyX4dHhvEUyDMMoJE0VdhqX4dFhVtywgkOHDwGwa/8uVtxQXp+l1FeKOtQwDKPlmNYjhMHtg8eUgc+hw4cY3D6Yk0SGYRjFZVorhN37d8fabhiG0crkqhBE5AQR+ZKI/I+I3C0iZyd5/oVzF8babhiG0crkPUJYB/yHqj4TOAO4O8mTDy0aYmbH5OV7Z3bMZGjRUJKXMQzDmBbkphBEZA7wcuBfAVR1QlUfTvIapb4SG8/dSM/cHgShZ24PG8/daA5lwzCMAHJbD0FEzgQ2Aj+hPDrYAaxR1Uer9lsBrABYuHDhWbt2OeVoMgzDMDxEZIeq9tfaL0+T0QzgBcAVqvp8yguiv696J1XdqKr9qtq/YEHN7K2GYRhGneSpEO4H7lfVH3r/f4mygjAMwzByIDeFoKq/Bn4pIn/gbVpE2XxkGIZh5EDeM5XfBQyLSCfwc+CCnOUxDMNoWXJVCKp6G1DT0WEYhmGkT97zEAzDMIyCYArBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbh0RIKYXh0mN7Le2m7rI3ey3sZHh3OWyTDMIzCkXcuo9QZHh1mxQ0rOHT4EAC79u9ixQ0rAGyhHMMwjAqm/QhhcPvgMWXgc+jwIdbctCYniQzDMIrJtFcIu/fvDtw+Nj6GXCZmQjIMw/CY9gph4dyFkb/7JiRTCoZhtDrTXiEMLRqquc+hw4cY2DrA/LXzTTEYhtGyTHuFUOor0d3V7bTv2PgYA1sHmP3R2aYYDMNoOaa9QgBYt3gdgjjvf3DiIANbB1h94+oUpTIMwygWLaEQSn0lVvavjH3cFSNXmFIwDKNlaAmFALD+nPWs6l8Va6QAsGFkg5mPDMNoCVpGIUBZKWxeutnZpwCgKIPbB1OUyjAMoxi0lEKAsvlo7yV72bJ0i7NiCJvLYBiGMZ1oOYXg4ysGvVTZsnQLx3ccH7rvvK55GUpmGIaRDy2rECop9ZU4+IGDrOpfFfj7gYkD5kcwDGPaYwqhgvXnrA80I00cmbDcR4ZhTHtyVwgi0i4iPxaRr+YtC8C+8X2B28fGx2yUYBjGtCZ3hQCsAe7OWwifqNxHNkowDGM6k6tCEJFTgHOAz+YpRyVRuY9slGAYxnQm7xHC5cAlwNGc5ThGrdxHNifBMIzpSm4KQUReB/xGVXfU2G+FiIyIyMiePXsykW3d4nWhv9mcBMMwpit5jhD+CHi9iOwEvgC8SkS2VO+kqhtVtV9V+xcsWJCJYFGjhFrrKxiGYTQruSkEVX2/qp6iqr3AW4D/VNWBvOSpZt3idczsmDlp28yOmU7rKxiGYTQjefsQCkupr8TGczfSM7cHQeiZ28PGczdS6ivlLZphGEYqiKrmLYMz/f39OjIykrcYhmEYTYWI7FDV/lr72QjBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAEwhGIZhGB6mEAzDMAzAFIJhGIbhYQrBMAzDAHJUCCJyqoh8U0TuFpG7RGRNXrJEMTw6TO/lvbRd1kbv5b0Mjw7nLZJhGEYqiKrmc2GRk4GTVfVWEZkN7ADOV9WfhB3T39+vIyMjmck4PDrMihtWcOjwoUnbBUFReub2MLRoiFJfKTOZDMMw4iIiO1S1v9Z+NUcIIrLQZVtcVPVBVb3V+34AuBt4SqPnTZLB7YNTlAGAUlaiu/bvYsUNK2zUYBjGtMDFZHS947a6EZFe4PnADwN+WyEiIyIysmfPniQvW5Pd+3fX3OfQ4UOsuamQ1i7DMIxYhCoEETldRM4D5orI6ys+A8CTkhJARGYBXwYuVtVHqn9X1Y2q2q+q/QsWLEjqsk4snOs2EBobH2P2R2fbSMEwjKYmaoTwHODPgROAN1Z8XgJclMTFRaSDsjIYVtWtSZwzSYYWDSGI074HJw4ysHWA+Wvnm2IwDKMpmRH2g6peB1wnIi9V1e8mfWEREeBfgbtV9ZNJnz8JSn0lvrf7e1wxcoXzMWPjY6y4YcWx4w3DMJoFFx/COSIyR0RmiMjXROQhEXlbAtf+I2AZ8CoRuc37LEngvImy/pz1bFm6he6ubudjDh0+xNuve7uNFAzDaCpqhp2KyG2qeqaInA/8GfA3wDdU9YwsBKwk67DTIIZHh1m2ddmxSKNarOpfxfpz1qcslWEYRjiJhZ0CHd7fJcDnVXUPOLaG05BSX4mV/Sud979i5ArzKxiG0RS4KIRtInIn8GLgZhGZDzyWrljFxjcjtYnbRG/fr2BKwTCMIlOzRVPV9wKvAs5S1cPAOLA0bcGKTqmvxNVvuJqZHTOd9je/gmEYRcdlpvIMyuGnW0TkC8DbgYfSFqwZKPWV2HjuRmeH81E9ysDWAVbfuDplyQzDMOLjYvP4NOW5B5u8z9mAeUk9Sn0l9l6yly1Lt3B8x/FOx1wxcoVNZDMMo3C4RBndXh1RFLQtC4oQZVSL+WvnMzY+5rRvZ3snm87bZPMVDMNIlSSjjI56uYb8E/cCR+sVbLqzbvE6Z7/CxJEJ8ysYhlEYXBTCJcC3ReQbIrId+Bbw3nTFal7q9SuYCckwjLxxWg9BRLqAZwEC/ERVx9MWLIhmMBlVMjw6zEU3XMSjhx912t9MSIZhpEHDJiMReauX2RRVHVfVW1V1B1ASkTcnKOu0pdRX4uAHDrKqf5XT/hNHJhjcPpiyVIZhGMFEmYzeC/x7wPYvUjYjGY7Emci2a/+uDCQyDMOYSlQLNSNkfYL9PJHOwnDEn8jW0RZ96wQxX4JhGLkQpRA6RWRKuIy3oM1x6Yk0fSn1lbjy/Csj5ysoaiuwGYaRC1EKYRPwRRE5xd/gff8ccGXagk1XfL/ClqVbQvcZGx+z2cyGYWROqEJQ1bXATcAPvTUQHgJ+AHxdVT+WlYDTlVJfiZ65PaG/bxjZYKYjwzAyJdLLqaqfUtWnAH8APFNVT1HVT2Uj2vRnaNFQ6G+KWsSRYRiZ4pS/WVUfVtXfpi1Mq1HqK0VOYLOII8MwssQtob+RGusWr0OQwN8s4sgwjCxxTX9dc5tRH1ErsJnZyGgWhkeH6b28l7bL2ui9vNc6Mk2KywjhR47bjDqJWnO5HrORVU4jS4ZHh1lxwwp27d+Fouzav6ulVwhs5voXlbriJBE5A+gSkT4ReZ73eSngls7TcCYs4iiu2cgqZ/Y0cwOQBIPbBzl0+NCkbYcOH2rJ0W2z17+oEcI5wKeAUygvkuN/PgB8KH3RWouhRUOBvoS4ZiOrnNnS7A1AEuzevzvW9ulMWP0LmmxaxI5E1DyEK1X1ZcBfqOrLVfVl3meJqn4xQxlbglJfCSU482ycimWVM1tMAcPCuQtjbS8CaTXGYfVsbHxs0jWK2pFw8SGcJCJzAERkg4j8SEQWpSxXSxJmNprXNc/5HM1YOZsZU8Dl0W31olAzO2ZGzrPJkzQb46h6VtlJKGpHwkUhrFDVR0TkTymbj1YBa9MVqzUZWjQUmPzuwMQB55e12Spns2MK+IlFoXrm9iAIPXN72HjuxsKu6xHVGDc6coiqZ5WdhKJ2JFwUgm/HWAxc6a2JkMj8BRF5rYjcIyL3icj7kjhnM1PqKzHnuDlTtsdZJ6HZKmcQRbSthmEKuEypr8TOi3dy9NKj7Lx4Z6Hft7BG1x8pNDJyiJpsWtlJKGpHwqVhv11EtgHnAjd52U5rL7NWAxFpp+ykXgw8G3iriDy70fM2O/vG9wVuj9NzaKbKCZMVwPy187nwKxfmaluNo5CmgwJuNcIa3XZpT8SME7SuenUnoagdiZpLaHoN91nAfaq6T0TmA6eq6o8burDI2cBHVPU13v/vB1DVj4Yd02xLaNZD7+W9gXMPeub2sPPindkLlDK+Pbe6IlaTVfmD5JnZMdMa+WlE2DMOewcF4eilR2NfY3D7ILv372bh3IUMLRqa8v647JMUrktouq6p/Bbg6ao6JCKnAid5pqNGBPxz4LWq+pfe/8uAF6vqO6v2WwGsAFi4cOFZu3ZN7/w+rdYghSnAauqplEnKM10VcqsS1BivuWkNY+NjU/adDs++4TWVK070KeCVwIC36VFgQ2PilU8dsG2KdlLVjarar6r9CxYsSOCyxcY3QVTaIbtmdKV6zTxt9q6msKxsq0V19hnJUm1WBXjksSkLRNLZ3pm6GadIPjMXH8JLVPUi4HcAqroP6Ezg2vcDp1b8fwrwQALnnRaMPz5+7PvY+FhqdvS846FdGnpBMrOtFtXZZ6TL4PZBDh89PGX77M7ZqY7M865/1bgohMMi0obXexeRbiCJsft/A6eJyFNFpBN4C/DvCZy36ckyRjnveOgg51o1imZmLiuqsy9titRLzYOwEWBYkEdS5F3/qnFRCJ8GvgwsEJHLgO8CH2/0wqr6OPBO4GvA3cC1qnpXo+edDkSFxSVdUfM2kQSZyKqJWlkuLXlaKWqoaL3UPMhrZJh3/asmKrndDABVvRr4IPAPwG+BN6rqF5K4uKpuU9XTVfXpqjq9u2AxiHoJk66oRTCRlPpK7L1kL6v6V03J55RH77zZwnYbpSi91DxHKXmNDItQ/yqJGiEcS3Gtqnep6jpVvVxV78xArpYmbMYyJF9Ri2QiWX/OejYv3dxSvfM88RvgsCivLHupeY9SXEaGaSisItU/iAg7FZEfq+rzM5YnklaYh+Azf+38wBA4SD4EM8t4aCN7gp4vUHP+hx9umcX7kXe4b60yphkOnsX9bXgegojcD3wy7EBVDf0tLVpJIbRd1haa/XQ6xEUb2RDWkHXN6ArtcPj7bDx3I9/b/T02jGyY9C6mMS8m7H3PYv6JS2Oft8JqlCTmIbQDs4DZIR8jRcJsiHFDMFs9eqRoZP08wvwDUcrAN5cAU5SBf3zS/oU8bekuPpS0nL9Fq59RayM/qKp/m5kkxiSGFg1N6bUIwsr+lc49s+qej2+XBcwklAN5PI+4DVZlj7f38t5E1uhwIeh9z8qW7tLYL5y7MHCE0IjCKmL9jBohBM0kNjIiyMm1eenmyPWXqylK9EjRyKtX5vo8kpQvrMHq7uqu6cyMavST7rnnGe7rMjpJw/lbxPoZpRBsEZycaTT8sWgxzkUgz2gWl+eRtHxhDdm6xetqNsBxzZaNKrK8wn1dGvt6FFat+1HE+hm1hGa6U/SM1ClCjHPRbKR59spcnkfS8kU1ZLUa4KCGMsxsmZQiq+d9SUIRuTT2cRSWy/0oQv2sxinbaVFopSijJMg7c2rQ9f0GJY7pK0nSjmaJCiF0eR55RtsE4RoSmUQUTj3va97veBgu9yNL2RNNf10UTCHEJ885BmGVQhA2L92cS4VNM3wwqIJ3tHUw57g57Bvfx8K5C1ly2hK23bst9Hk0a3hjEoosapJcz9yewHe3qPfL9X5kVT+TmIdwKvAJ4CnATcAnVPWw99v1qnp+gvI6YQqhuSjiXIo0e2Uuazs0a4+3muqG7ODEwYbXEoh6XyD4PhRtROVTNEWVxDyETcAtwLuAk4FveZlOAbLLNmY0LVG20LwcZ2lGs7iUqZY/oBmS6wXZxx957BE62ydnxY8bhVPLdh5077Kyw8f1UxQtJYUrUQphgapuUNXbVPVdwHrg2yLydBJYU9mYHkRVlKFFQ1OS1fnk6ThLK5rFtUy1FEfQ4i1ZO+ajnmuQ4/vw0cPM7pzdkCJzSYVefe9cGt5Gnc71OMybQbEHEWUyugs4S1V/V7Ht1ZRXSzteVU/ORsQnaFWTUVFzDbmYN1bfuDqT1AfVcuVxv9JYHzoPE1ItX0iYWScJM43/7KJ8CdX3rlFHfi3imH+KWleT8CG8G7hVVb9Vtf35wFpV/ZNEJI1BKyqEsEamu6ubdYvXxXrZkn5ZXStKlpXExbGb9vX9ss7smMmjhx+d9HuajVFSuK5zXU2SMiWlCJO4f3EcxEX1/zTsQ1DVf6xWBt72HwM3Niif4UjQ8BziL6uZxoQs14k1USYa1+G8635h5oyx8bFMJqL5Zd28dPOURkQQlp+xPFbjkMfkpXrPfXDiYGL3NSmTSxL3z9VPUcSZx3FxWTEtiL9JVAojlKgX99DhQwxsHXCyi6bxssZx6AU16K5KKo4yS8KxmwRB91tRtt27LdZ5wu7xvK55dctW7zVrkfTa30n4epJwOrs6iKOUT9EmaIZRr0KwPEcZ4fLiuvR60+hpulaUsAZ9zU1rnJRUHGWWlGO3UZK632GLJR2YOJDaLF4X5y5Au7RP2Va0HnES0T6uo5Uo5V39/l9w/QXMXzu/cAqiXoVgUUYZ4Vo5a1VEl55S3MbDtaLETcFc3WjGaVxd71faYYlhPfi41y31lZhz3Jwp2yeOTNRseOs1E1Y/1+6u7sCQ0iN6JPD4IuXKinpH47zvLqOVMOUD5GrGjEOUU/kAwQ2/AF2qGpU6OxVa0akM5Yp90Q0XTXFQVhMV5VHL4ZWmQ6zWhKNqqh1+cR2DlY7deV3zODBxgIkjE8d+T9rRF+bIFpFErlvv5KskHdJBgQFh0UB5zxJ2Ia33Peg+Ldu6zOn9T/O+JeFUnq2qcwI+s/NQBq2OywsV1fus1ZuPMss0av+M0yvubO+cMpyPO+yv7M3tvWQvm87blGo8eFpx+T712sGTNBMG9ZAbMcfkbVNPywEcdJ+KYsZ0wXIZNQFJpESoRVQvfmbHzIZ6UmFJ7oKu193Vzd5L9gaeI+/47jAZskiYV09vNouQ1XqeSxHCM9N4ZmH3Io35KXFxHSFYT78JiOo5CJJIAxm2IlS7tEf2pFwag8pRiL9vmILbNx6cdd1P15wXUatbpbGaViVB98/leWexClk9zyWqd57kM45SVkk/M5fVz3wTW1BnqChpLWyE0ASk1dNzsbVH9WoaGTkULfkXRDcgUfKGNbxFmJBUhJFVNVkkpMvaZ1Zvumuob5JpXJJIbmcUhDQSZVVHoIyNj6GqdHd1T7J598wNzmPYJm0N2WCLkvzLt2XLZcKyrctCI3Ki7PEu0VZ52czzWoUsiiwS0tXyESSda8jFXxM2yXRW56xCPBfISSGIyCdE5H9E5A4RuU5ETshDjmYhjURZYY7QWZ2zJiVVOzhxMPD4oxrck3N1jBUh+VelUoSpjvvKBqRWI1ZrNnZey3bGJQvFlUVnwKWBTlJZuii5Ii6ZWU1eI4Sbgeeq6vOAnwLvz0mOpqEyJQLAsq3LGqqwtV5OvxELmysQhl8BXBqWvHuvYT22Snxl0UgjFtZbdZ1lnhVZKS6/M9Dd1X1sW9eMrkSvkfXylC7vRxGXzKwmF4Wgql9X1ce9f38AnJKHHM1GWIVdfePq2L26Wi+nS2NZjV8BmqVH7NIzE4Th0eGGRjRR12nkGSZN1rl4xh8fP/Y96bQXWZskXd6PophJo8jdqSwiNwDXqOqWkN9XACsAFi5ceNauXfGzME4XopakjJteupZTzXUyWbu0c1SPTnJYFtFhHIRrVs+4cruuJlZJPc8waeI6extxWBc1JDZt8pIp9zWVReQbwO8H/DSoql/x9hkE+oGl6iBIq0YZ+cSZ8etSseqJqqnGXx/ZJaQ072UNq3GND48jt+usZRfC5mSkRdy8//VG6QyPDjOwdSDwt6K9I9UUUcm4kHuUkaq+WlWfG/DxlcFy4HVAyUUZGPFsjS7mkCgbvmtOoKDEXUVcJS2I6mF+ULI2iJfvqdas5TiMjY9lajqKY9JwNS9V36/VN64+Fp8fRJu0pWI+S8JZ3iym0EbIxWQkIq8FPgm8QlX3uB7X6iOE4dHhTPOiuMxT6JrRFWgOKYIJJC5JxK7XMru4jkp8sjazufaAo0arPXN7Qt+ZsBnqUdQzM76yDEtOW8JVt1+V2mI7QWbTopG7ySjyoiL3AccBfkvyA1VdWeu4VlcIAHJZ7czjaTW+cRN3+Q1DkStKNfVOTqtc+9hlglKYyaSaoppQXP1ZSeGqGOOkSYmrbF1MtkXt+ORuMopCVZ+hqqeq6pnep6YyMMrUMju0S3tqL2SlicnPdhlWQdqlvemUAUSb0VziyIPMLh1tHRycOHjMXAG1n6NP0cxsPkHlTEsZgHusftjCRI2c08flWaQRlZXlpEabqdxkRNn2Z3bM5Ko3XJV641s9oSuII3rkmJ11YOsA89fOb3pbq0scedBaAiIyJff9ktOW1PTRFC0ksZKgMEtXZRDmY4oijYyhruf0G+Qo/1i9MrhcO0u/hSmEJqOyIsITq1b1zO1h+RnLGdw+mNj6xGFEzVEIc8wmHWeeB65O18pRxqzOWVOiiw4dPsS2e7dNaVBX9a/KdeZ2XKpHUy6jnpkdM1nZvzJy8Z2gY5actsTpnQ1r5KsbckFYctqSmvIGzWb3z+UShNAoWc8NyX0eQhzMhxBOmO10Zf9K1p+zPnK/mR0zWX7Gcrbdu83JzBPlOIXotRuKNhfBJ8h3AFMzjAZti2q0s0jkVhRqOcyDkrgNjw6z5qY1xwITuru6edNz3jTpXYzjFA57v88+5Wz+8xf/GTvQIe+khkm9P4V2KteLKYRwopx8m5durukYjRMVFFVJgEhTUhEbwjRXPGuWSXpJMTw6zPLrlgcur1k9r8J1LkMjK+Y1urqbS9RYmvMSknp/Cu1UNpInzG6p6KThZdR+lUQNS6NMJ7XmLxTRSRo2dyDI1BN3BblmSFeQJKW+Umjiw+p5Fa5LXkiEAAAXEElEQVTmkLhJ4YICA+pNLNdIUsNq6jHVZv3+mEKYJkQ1tJUvfRKT26LytgQlLvMpakMYxwnoO/VcnXxFyOqaNVHvmEvnpHp7Eknh6j1HUg1yvc7hrN8fUwhNTmUERBiVL31YuGDYcWG9mqieUamvxN5L9rJl6ZamaAjjNCy1VpALopGsrnmvPexCtYxRzlqXzkn19iQa5XrPkVSD3IhzOMuswOZDaGJcZr0G2WRdZ3IuP2N5IjM8q2XOMheMy/Xizh4OIg3fSBHWHq5FmIxt0ha4lkatFcSinMWNvjd55iHKO7jAnMotQK2RgR/VAU9ExszrmgeU1y6uXgg8KUdcGFk3cPU2OG3SFuoUndU5KzMncRZLpzbaMIbJ2N3Vzfjj4zXvfbMmi4tL3sEFphBagFpT6cNC4yqJapCT7tUkXSlqNSb1Xi9KkQCZKbWk7391iKdPI/JHyVidBXe6NvYu5D3asyijFqCW7Xv3/t01F7qJsmMmvcJTkksIrr5xdeQayI1cz8VpnoVvJMn7H7UCXiMTncJkaZM2lm1dBsDmpZsLs55zXjRLcIEphCbGJcTTpbEN2yfpkLekGrjh0WE2jGyoGSrbyPWC8jZV5iLKwsmX5P2v1TGoN91C2DtYmbqk2WeoJ0XeS8a6YAqhiXEJ8XRp/NqkLXTN4yR7NUk1cFFJ9Wolmqu8nksET5458JO8//XG28eVMSidQ5qpFoxkMR/CNCEocujau66tuXSjT1b2zCSciLVy8Vf6B8Kul9Ys2aISFYCQZGRP3tE0RjDmVG5hhkeHueD6Czh89PCU347vOJ7xx8cDZ5NWhwTGbQySjhgJO59rmo4oXBv66dLAhYXWBuUXCtu/0dw/9QYOVOc6CpLXiMacyi3M4PbBQGUAMH/mfMI6Ab5ZoR4zSdKmlajzhU2uW9m/0rmhyHKWbBEIMj9tWbqFvZfsDbxn9U6kSnJm7/y18xnYOjBplDs2PsaFX7nQfBIpYQphGhJlL/Z720H42+tpDJJO0xt1vqDGbfPSzZOyutYiy1myRSGOUzPsHdq1fxdymTDjb2cgl8kU30sSfo+oiCiAiSMTifokijYbPE95ZmR2JSMzFs5dGGovVpSDEwfpbO+cksnTb+TqCddMMqTU5Xx+CGi9hKUuDlrbAILTYPde3jttY+yj3iHg2MQ9f+QGT9yrRp9NrYgoqP+9qjVLP6g8WVJtqstaHhshTEOGFg3R0dYR+vvY+Biqeiw6yc/P42fyrMdM4nJMnJ5PVHx79fH19KiCFhqqvAfV+1b2rIHcIo+SJuze1QppriTOSNDlWbk09o3Mxah8bhtGNmS6AE0tsl4QpxpTCNOQUl+JK8+/MjAc1cf3MczsmDmltxe0vGMtM4lLiGecRtQ1vn31javrbpxLfaVj16m+B1HH511pkyLqmVSbfmrh0oi7vgO1GvvO9s7Ad7GWsklzveWkSHqkHRdTCNMUP+OoXqqhFXpsfCywYQta3rGWHbiW7ThuIxo1x6Ly+I07NjbUONfTuOddaZMirOwDWwemTMCrtTymS4/d9V5HjU66u7rZdN6m0KioemauB5FX0EDeQQymEFqAuC/T7v2765pVGXVMvY3oI489Evl7UBI6l/M2IlfelTYpospY3aBGNdKuTnbXex0WEaWXakNRUa7rLacdNBA1ksk7iMEUQgsQ9pKF9b4badjCXvZ6GtGo8FmfNgl+hV3L4CpXZbl8p3wlRY88Cnoute5RZYMa5HMBYkURxXkH4nZIXJRNWD1Y2b8ysxxDtUYyeec8solpLULYIvJJZmBMMkvo8OgwA1sHnK47o20Gjx99vK4yuEzACltzec5xc6akES8iYWUMWu+imiQn4KWZ8dN1Qlze6bbzmvneFDOVReQ9wCeABaq6t9b+phCSJ4vc+P7L7nqtRhasCVoDola50kqjnRZxn1mU/FHrXvj7JFnGtBrkvNNLu5LXzPfCKwQRORX4LPBM4CxTCM1BVIVO6mWvtfBPFGms9Fak9BX1NHwu8jdLgxpF3r1/F4o+QsjTh/CPwCUQscKLUShq2T+TcrY2Eq2TRORRNVHlynpWaT1RUS7PJW/bdRI0Q3rpvJ3GtchFIYjI64FfqerteVzfiCaskavVGCX1soc1YN1d3U4x8Y1GHlUTVq4lpy3JfIJaPVFRrs+lGRrUZqfoijc1hSAi3xCROwM+5wGDwIcdz7NCREZEZGTPnj1piWt4RI0CXNJJJPGyhzVg6xavY2X/yppKISgnPySX898v17Z7t2U+Qa2eUViQ/MvPWD5p0Z9mnGXdrBRZ8WbuQxCRPmA74NekU4AHgBep6q+jjjUfQvpE2TiBzOyfUfZg/7dd+3chyCT7eBo+hDDy8C0E2fr9TK/Vyf0aXQvCmD4U1oegqqOqepKq9qpqL3A/8IJaysDIhqhRQJb2z6helP+bXqpsXrp5Ss99/TnrMxmWh/XKFU2t113qK7H8jOWTRkmKctXtV026XtRIb7qk3oiiaBlMm4Xc5yGIyE6g36KMikFSoaOtQK3w2EZ73XEXCKocqYXt093VHZpWOs9Ff5J8r+IuBtQKFHaEUI03UqipDIxsqDUK8BPCLZy7kN37dwdmBy0iafQYq2fvVtNIr7sRX07190qillTNK/VG0osrhaXPHhsfS8XpP51GI7krBKNY1HIM57nofL2kKbNvvgpzdLtmAa1uUKLMOi6O5biNe56hj42YsILuXdQ9T9o01oz1IYrcTUZxMJNR/iQ1sSZL01MWk4HqvUaYgzfMDOWvG13LKbz6xtVsGNkQmt65mi1Lt+RmSqnXOR9277pmdEWOhOKYxpptFnsYTWMyMpqLJFI/p9mritNjTDJddb0O97DecVTorMso7qrbr5rUyArCrM5ZgefsmduTq1293gmNYfcOiFzcZ17XPCcTTyMptSu3N5NJyRSCEYuwSupaySC9BWbCKvC8rnmB+ydpM693DkZYg3JEj9T05YRFYYUtBHNc+3GFnCUbV5n6DWxYepN94/tC19LoaOvgwMQBp85IIym1/e1xOz95Kw9TCEYsgipvnEoG6S0wE6fHmEZDWM+Eo7AGxVco9YTOht1Hv6Es2izZOMq0soENw0+JvveSvWxZumXSeeccN2fSWuIQ3hlpJKW2/27F6fwUwR9hPgQjNtV21YMTBwNttmF21LTsrlG26M1LNxcyXLbRSWKVk/TapZ0jeuTY32qaJTtrFK6JD8PuYRx/RRIptdO4Xj2YD8FIjeqe8L7xfYH7hfWw0prgFjV8L2q6gEbSfVT3ln0lEKQM4tzfJM0WSfd6XUeRhw4fYvl1y53Xaw7aHicHVFgodpzrFWFpVlMIRsPEdQqmleArDUWThU23XmUVFm/v0y7tDSmZtOYEpJF5NogjemSK7HHeEdf3NOqexbleEZZmNZOR0TBFyo2T9ozXIuX8CTNH+NQz8zhps0XS+Z7qWTwprVXTKs11UddtZGGopN63wi+QUw+mEIrLdExpUfQY8/lr50fG29cjZ9INeFQKjb2XxEtQEJXUMIo0UnK4KKZ6rptWPTIfgpEpjdro8w63C6IINt0whkeHeeSxR0J/r9dn4EfoVFMZRhnnOQ0tGqKjrWPK9rHxMVbfuNpJPv+6lf4SRelo6zi2RkZ3V3dN2cPOW10elzLWMtf51417v/L2dZlCMDKnupKsvnF1w3bruBXPZf8i2HSr8eUe2DrA4aOHA/dpxGcQ5ZCux79Q6isx57g5gb9tGNng/IyDGmC//EcvPcreS/Zy9Ruujj2fobo8F1x/ARd+5cLIMg6PDteMdKpnAaUidIpMIRiZElQJN4xsaMjxGHTOZVuXhfZAXRu2pJ3ULhU+ah+XGHyfZVuXOTUqYT3dIId0vQ7isCg0RZ2fcVSyPr+McYMVwpRM1DyF4dFhLrj+gkhZ61lAafWNq1m2dVnuOZHMh2BkimscObjbYMPO6c8/qG4Q4vgG6rXpVh+35LQlNRftqeVUjHPvwq5RTRyfQb3+hSi5G33GkLzDOwhfzlplqXzfXO/X8Ogwy7YuC5WlZ25Pw74E8yEYhSSO/d3VNBN2TkVZft3ySaapqAoddJ56bLpxRkEDWwdqZjj14+njKgP/+KheeByzWL0mtKFFQ6HZYF2fcdSorF6fThzTn5/+JOpaik56P1zv1+D2wUjFlOVowRSCkSlhlaS6wYhjmomq2Ef0yLFG+YqRKyIb1aR8A2G5hMLwK3yYbEf0SE0zRRRRjVgcs1i9JrRSXylwLew4z7jUVwrMTQT1P7ewNCxBiQUPTBxgeHQ48lrV62K43i8XhZbVinamEIxMCaskK/tX1j1RLaoH6ko9voEwe389PdaoDKdAqAPZhahGLI7dvZEJhevPWR+43GkcM8i6xesS9ekElefK86/khCedMGXfiSMTDG4fDI2a6mzvDJzB7HK/Gh0JJ4n5EIzMSSPWOm7+/0rqsdFG2fvDJiu5xM53tndOcWo2Sp5rHSRNFvNdonwLeqkyPDrMmpvWHJsD0ujSnK6T7dqlnavecFVd17GJaUbLUdlYtElbYAhlNfU6JKMc00OLhgKVxfIzlrPt3m2RZquOtg6O6BGOqtuEpu6ubsYfHw9tTOqZANbqxA1SSILKd3de1zwOTBwI7BjUO3PZnMpGy1HpAL7qDVdFLpICjZkboiathZkK1p+znp0X72TL0i2hsh0+epgTn3RioG27s71zivzrFq8Lzf3v/27EI8wEGSdMNi6V7+7eS/ay6bxNgSbEtH0JphCMaUlQo7yqf1ViCfVqRZBERSf5soURtG7BledfyabzNgXKX+orBeb+L0rOpWaj1FcKNRllNUu91FcKHSWmKYOZjAyjDpJIRFb0XEmtTBGeTZIymMnIMFIkiRTeaa0LYTROEZ5NHjLYCMEwcmQ6ZomdLhTh2SQlg0UZGYZhGEATmIxE5F0ico+I3CUia/OSwzAMwygzI4+LisgrgfOA56nqYyJyUh5yGIZhGE+Q1whhFfAxVX0MQFV/k5MchmEYhkdeCuF04GUi8kMR+ZaIvDBsRxFZISIjIjKyZ8+eDEU0DMNoLVIzGYnIN4DfD/hp0LvuicAfAi8ErhWRp2mAh1tVNwIboexUTktewzCMVieXKCMR+Q/KJqNbvP9/BvyhqkYOAURkDxA/KXyZ+cB0SepiZSkmVpZiYmWBHlVdUGunXJzKwPXAq4BbROR0oBOHQroUKAwRGXEJu2oGrCzFxMpSTKws7uSlEDYBm0TkTmACWB5kLjIMwzCyIxeFoKoTwEAe1zYMwzCCaaVcRuHpJZsPK0sxsbIUEyuLI02VusIwDMNIj1YaIRiGYRgRmEIwDMMwgBZRCCLyWi+R3n0i8r685QlCRHaKyKiI3CYiI962eSJys4jc6/090dsuIvJPXnnuEJEXVJxnubf/vSKyPCPZN4nIb7yoMX9bYrKLyFnevbnPO3bq+obpluUjIvIr79ncJiJLKn57vyfXPSLymortge+ciDzVm6F/r4hcIyKT18VMtiynisg3ReRuL4nkGm970z2biLI03bMRkSeJyI9E5HavLJdFXV9EjvP+v8/7vbfeMtZEVaf1B2gHfgY8jfJ8h9uBZ+ctV4CcO4H5VdvWAu/zvr8P+Lj3fQlwEyCUZ3v/0Ns+D/i59/dE7/uJGcj+cuAFwJ1pyA78CDjbO+YmYHHGZfkI8J6AfZ/tvU/HAU/13rP2qHcOuBZ4i/d9A7AqxbKcDLzA+z4b+Kknc9M9m4iyNN2z8e7VLO97B/BD734HXh9YDWzwvr8FuKbeMtb6tMII4UXAfar6cy2Hu36BcqbVZuA84Crv+1XA+RXbr9YyPwBOEJGTgdcAN6vqPlX9LXAz8Nq0hVTVbwP70pDd+22Oqn5fy7Xg6opzZVWWMM4DvqCqj6nqL4D7KL9vge+c13t+FfAl7/jK+5I4qvqgqt7qfT8A3A08hSZ8NhFlCaOwz8a7vwe9fzu8j0Zcv/J5fQlY5Mkbq4wusrWCQngK8MuK/+8n+kXKCwW+LiI7RGSFt+33VPVBKFcIwE8THlamIpU1Kdmf4n2v3p417/TMKJt8Ewvxy9INPKyqj1dtTx3PzPB8yr3Rpn42VWWBJnw2ItIuIrcBv6GsYH8Wcf1jMnu/7/fkTbwdaAWFEGTTLGKs7R+p6guAxcBficjLI/YNK1MzlDWu7EUo0xXA04EzgQeB/+9tb4qyiMgs4MvAxar6SNSuAdsKVZ6AsjTls1HVI6p6JnAK5R79syKun1lZWkEh3A+cWvH/KcADOckSiqo+4P39DXAd5ZfkIW9YjvfXXzcirExFKmtSst/vfa/enhmq+pBXgY8C/0L52UD8suylbIaZUbU9NUSkg3IDOqyqW73NTflsgsrSzM8GQFUfBm6h7EMIu/4xmb3f51I2aybfDqThNCnSh3J6jp9Tdrr4Dpbn5C1XlYzHA7Mrvv8XZdv/J5js/FvrfT+Hyc6/H3nb5wG/oOz4O9H7Pi+jMvQy2RGbmOzAf3v7+o7LJRmX5eSK7++mbLcFeA6TnXo/p+zQC33ngC8y2XG4OsVyCGW7/uVV25vu2USUpemeDbAAOMH73gV8B3hd2PWBv2KyU/naestYU7Y0K1ZRPpSjJ35K2U43mLc8AfI9zXtotwN3+TJSthNuB+71/vqVUIBPe+UZBforznUhZefSfcAFGcn/ecrD9cOUeyd/kaTsQD9wp3fMp/Bm2GdYls2erHcA/17VCA16ct1DRYRN2DvnPesfeWX8InBcimV5KWVTwR3Abd5nSTM+m4iyNN2zAZ4H/NiT+U7gw1HXB57k/X+f9/vT6i1jrY+lrjAMwzCA1vAhGIZhGA6YQjAMwzAAUwiGYRiGhykEwzAMAzCFYBiGYXiYQjCaDhHprshu+euqbJdOGSpF5EoR+YMa+/yViJQSkvm7XvZJX85rkjhvxfnvF5ETkjyn0XpY2KnR1IjIR4CDqvoPVduF8vt9NBfBqhCR7wLvVNXbUjr//cBztTzz1TDqwkYIxrRBRJ4hIneKyAbgVuBkEdkoIiNe3vkPV+z7XRE5U0RmiMjDIvIxLz/990XkJG+fvxeRiyv2/5iXx/4eEXmJt/14Efmyd+znvWudGUPmLSJyhYh8R0R+KiKLve1dInKVlNcauNXPbeXJ+49eOe8QkdUVp7tYRH7sbT+94RtqtBymEIzpxrOBf1XV56vqryinaOgHzgD+RESeHXDMXOBbqnoG8H3Ks3KDEFV9EfBewFcu7wJ+7R37McpZOMO4psJk9LGK7acCrwDOBTaKyHHAXwMTqtoHLAM2e+awVcCTgTNU9XmUUxv7PKSqzwc+C/xNhByGEciM2rsYRlPxM1X974r/3yoif0H5XX8yZYXxk6pjxlX1Ju/7DuBlIefeWrFPr/f9pcDHAVT1dhG5K0K2N4eYjK71TFv3iMgvgdO8837CO+9dIvIA8Azg1ZTz+Rzxfqtcu6FSviUYRkxMIRjTjUf9LyJyGrAGeJGqPiwiWyjnhalmouL7EcLrxWMB+ySxZGS1Iy8shbF/vTDHX5B8huGMmYyM6cwc4ADwSMXKX0nzXeBNACLSR3kEEpc3SpnTKZuP7gW+DZS88z6L8hKS9wFfB1aJSLv327yGS2AYHtaLMKYzt1I2D91JOR3w91K4xj8DV4vIHd717qS8olUQ14jIuPf9IVX1FdR9lBXAScAKVZ0QkX8GPiMio5Qzr77d2/4ZyialO0TkccoLxGxIoVxGC2Jhp4bRAN6CJTNU9XeeierrwGn6xFKItY7fAnxJVa9PU07DcMFGCIbRGLOA7Z5iEOAiV2VgGEXDRgiGYRgGYE5lwzAMw8MUgmEYhgGYQjAMwzA8TCEYhmEYgCkEwzAMw+N/AVVIz1BvdO/NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd88569e198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "\n",
    "# start the session\n",
    "with tf.Session() as sess:\n",
    "    #    initialize the variables\n",
    "    sess.run(init_op)\n",
    "    mc_losses = []\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        \n",
    "        current_start = epoch*batch_size\n",
    "        data_train_batch = data_train[current_start:(current_start+batch_size),:]\n",
    "        start_train_batch = start_train[current_start:(current_start+batch_size),:]\n",
    "        \n",
    "        # begin in state of all zeros\n",
    "#         begin_state = np.zeros((batch_size, state_size))\n",
    "        # begin in random state\n",
    "#         begin_state = np.random.rand(batch_size,state_size)\n",
    "\n",
    "        begin_state = start_train_batch\n",
    "\n",
    "        _,cost,est_data_batch = sess.run([optimizer, estimation_cost, est_data_series_2], \n",
    "                                          feed_dict={batch_data_placeholder: data_train_batch, \n",
    "                                                     init_state: begin_state,\n",
    "                                                    batch_start_placeholder: start_train_batch})    \n",
    "\n",
    "        if epoch % 100 == 0: \n",
    "            \n",
    "#             begin_state = np.random.rand(batch_size,state_size)\n",
    "#             begin_state = np.zeros((batch_size,state_size))\n",
    "            begin_state = start_test\n",
    "            mc_cost, mc_estimation, l1 = sess.run([estimation_cost, est_data_series_2, layer_1_series], \n",
    "                                             feed_dict={batch_data_placeholder: data_test, \n",
    "                                                        init_state: begin_state,\n",
    "                                                       batch_start_placeholder: start_test})\n",
    "            \n",
    "            \n",
    "#             plt.plot(epoch, mc_cost, 'bo')\n",
    "#             plt.plot(epoch, zero_force_error, 'ro')\n",
    "#             plt.plot(epoch, mse_error, 'go')\n",
    "            print('Epoch {}, Train Cost {}, Test Cost: {}'.format(epoch, cost, mc_cost))\n",
    "            plt.plot(epoch,np.log(cost),'go')\n",
    "#             print('Epoch {}, Train Cost {}'.format(epoch, cost))\n",
    "    \n",
    "    with tf.variable_scope('onelayer', reuse=True): \n",
    "        weight1 = tf.get_variable('kernel').eval()\n",
    "        bias1 = tf.get_variable('bias').eval()\n",
    "    \n",
    "    plt.xlabel('Training Epoch')\n",
    "    plt.ylabel('L2 Test Cost')\n",
    "    plt.title('NN Equalizer: no noise')\n",
    "    # plt.text(1000, .025, r'NN equalizer')\n",
    "    # plt.text(0.5, .025, r'Zero Force equalizer')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 4.9910903  7.0042305  9.021561  11.041863  13.065231  15.094739\n",
      " 17.128473  19.16885   21.215706  23.270695  25.335497  27.405947\n",
      " 29.488989  31.57713   33.67795  ]\n",
      "[ 5.  7.  9. 11. 13. 15. 17. 19. 21. 23. 25. 27. 29. 31. 33.]\n",
      "[-0.0089097   0.0042305   0.02156067  0.04186344  0.06523132  0.09473896\n",
      "  0.12847328  0.16884995  0.21570587  0.27069473  0.3354969   0.40594673\n",
      "  0.48898888  0.57712936  0.67795181]\n"
     ]
    }
   ],
   "source": [
    "k=7\n",
    "\n",
    "print(np.array(mc_estimation)[k,:])\n",
    "print(data_test[k,:])\n",
    "\n",
    "print(np.array(mc_estimation)[k,:]-data_test[k,:])\n",
    "\n",
    "# print(np.array(est_data_batch)[k,:])\n",
    "# print(data_train[k,:])\n",
    "\n",
    "# print(np.array(est_data_batch).shape)\n",
    "# print(data_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
